#! /bin/sh
. /usr/lib/innshellvars

##  Daily news maintenance.
##  Optional arguments:
##      delayrm              Delay unlink files, then do it quicker (expire -z)
##      expctl=xxx           Use xxx as expire.ctl file
##      expdir=xxx           Directory in which to build new history file
##      expireover           Runs expireover after expiration
##      expireoverflags=xxx  Pass xxx flags to expireover
##      flags=xxx            Pass xxx flags to expire
##      lowmark              Create and use a lowmark file
##      mailto=xxx           E-mail address to which to send the report
##                                                        instead of newsmaster
##      noexpire             Do not expire
##      noexpireover         Do not expireover
##      noexplog             Do not log expire output
##      nologs               Do not scan log files
##      nomail               Do not capture and mail output
##      noprocbatch          Do not run procbatch on innfeed dropped files
##      norenumber           Do not renumber the active file
##      norm                 Do not remove certain old files
##      norotate             Do not rotate logfiles
##      nostat               Do not run innstat
##      notdaily             Not a daily run, and therefore implies nologs
##      postexec=xxx         Path to a program to run after expiring
##      procbatchdir=xxx     Directory in which to run procbatch
##      tmpdir=xxx           Working directory for `sort' and such
##      /full/path           Path to a program to run before expiring

EXPLOG=${MOST_LOGS}/expire.log
INNSTAT=${PATHBIN}/innstat
INNFEEDCONF="${PATHETC}/innfeed.conf ${PATHETC}/innfeed-delayed.conf"
HOSTNAME=$(hostname)
DATE=$(date)
TAGGEDHASH=${DO_DBZ_TAGGED_HASH}
ADDHEADER="Auto-Submitted: auto-generated"

##  Try to find the path to innfeed's spool.
PROCBATCHDIR=
if [ -d "${PATHSPOOL}/innfeed" ]; then
    PROCBATCHDIR="${PATHSPOOL}/innfeed"
fi
for F in ${INNFEEDCONF}; do
    if [ -f "${F}" ]; then
        INNFEEDSPOOL=$(${AWK} '{gsub(/:|#/, " & ")} \
                               {if ($1 == "backlog-directory" && $2 == ":" \
                                    && $3 != "innfeed") \
                                    print $3}' ${F})
    fi
    for DIR in ${INNFEEDSPOOL}; do
        if [ -d "${PATHSPOOL}/${DIR}" ]; then
            PROCBATCHDIR="${PROCBATCHDIR} ${PATHSPOOL}/${DIR}"
        fi
    done
done

##  If your expire does not pause or throttle innd, enable this next line:
#MESSAGE="Expiration script $$"

##  Renumber all at once, or in steps?  Set to the delay if steps (that
##  may take a long time!).
RENUMBER=

PROGNAME=news.daily
LOCK=${LOCKS}/LOCK.${PROGNAME}

##  Set defaults.
DAILY=true
DOEXPIRE=true
DOEXPLOG=true
DOEXPIREOVER=false
DOLOGS=true
DOMAIL=true
DOPROCBATCH=true
DORENUMBER=true
DORM=true
DOSTAT=true
EXPIRECTL=
EXPDIR=
EXPIREFLAGS="-v1"
EXPIREOVER=expireover
LOWMARKFILE=
RMFILE=

HISTDIR="$(dirname ${HISTORY})"
TOUT=600 # timeout value for ctlinnd commands
if [ -z "${HISTDIR}" -o X"." = X"${HISTDIR}" -o ! -d "${HISTDIR}" ]; then
    if [ -d "${PATHETC}/hist" ]; then
        HISTDIR="${PATHETC}/hist"
    else
        HISTDIR="${PATHETC}"
    fi
fi
if [ "${ENABLEOVERVIEW}" = "true" ]; then
    DOEXPIREOVER=true
fi

EXPIREOVERFLAGS=
PROGRAMS=
POSTPROGRAMS=
REASON=
SCANARG=
DOGROUPBASEEXPIRY=false

if [ "${GROUPBASEEXPIRY}" = "true" ] && [ "${ENABLEOVERVIEW}" = "true" ]; then
    DOGROUPBASEEXPIRY=true
fi

##  Parse JCL.
for I; do
    case "X$I" in
    Xdelayrm)
        RMFILE=${MOST_LOGS}/expire.rm
        ;;
    Xexpctl=*)
        EXPIRECTL=$(expr "${I}" : 'expctl=\(.*\)')
        case ${EXPIRECTL} in
        /*) ;;
        *)
            EXPIRECTL=$(/bin/pwd)/${EXPIRECTL}
            ;;
        esac
        ;;
    Xexpdir=*)
        EXPDIR=$(expr "${I}" : 'expdir=\(.*\)')
        ;;
    Xprocbatchdir=*)
        PROCBATCHDIR="${PROCBATCHDIR} $(expr "${I}" : 'procbatchdir=\(.*\)')"
        ;;
    Xtmpdir=*)
        TMPDIR=$(expr "${I}" : 'tmpdir=\(.*\)')
        ;;
    Xexpireover)
        DOEXPIREOVER=true
        ;;
    Xexpireoverflags=*)
        EXPIREOVERFLAGS=$(expr "${I}" : 'expireoverflags=\(.*\)')
        ;;
    Xflags=*)
        EXPIREFLAGS=$(expr "${I}" : 'flags=\(.*\)')
        ;;
    Xlowmark)
        LOWMARKFILE=${MOST_LOGS}/expire.lowmark
        DORENUMBER=false
        ;;
    Xmailto=*)
        NEWSMASTER=$(expr "${I}" : 'mailto=\(.*\)')
        ;;
    Xnotdaily)
        DAILY=false
        DOLOGS=false
        ;;
    Xnoexpire)
        DOEXPIRE=false
        ;;
    Xnoexpireover)
        DOEXPIREOVER=false
        RMFILE=
        ;;
    Xnoexplog)
        DOEXPLOG=false
        ;;
    Xnologs)
        DOLOGS=false
        ;;
    Xnomail)
        DOMAIL=false
        DOSTAT=false
        ;;
    Xnoprocbatch)
        DOPROCBATCH=false
        ;;
    Xnorenumber)
        DORENUMBER=false
        ;;
    Xnorm)
        DORM=false
        ;;
    Xnorotate)
        SCANARG="${SCANARG} norotate"
        ;;
    Xnostat)
        DOSTAT=false
        ;;
    X/*)
        PROGRAMS="${PROGRAMS} ${I}"
        ;;
    Xpostexec=*)
        POSTEXEC=$(expr "${I}" : 'postexec=\(.*\)')
        POSTPROGRAMS="${POSTPROGRAMS} ${POSTEXEC}"
        ;;
    *)
        echo "Unknown flag ${I}" 1>&2
        exit 1
        ;;
    esac
done

##  Setup mail subject and command.
if ${DAILY}; then
    MAILSUBJ="${HOSTNAME} Daily Usenet report for ${DATE}"
else
    MAILSUBJ="${HOSTNAME} intermediate Usenet report for ${DATE}"
fi
if ${DOMAIL}; then
    MAIL="cut -c 1-1022 | ${SED} -e 's/^~/~~/' | \
      ${MAILCMD} -a '${ADDHEADER}' -s '${MAILSUBJ}' ${NEWSMASTER}"
else
    MAIL="cut -c 1-1022 | ${SED} -e 's/^~/~~/'"
fi

##  Sanity check.  Shouldn't we just bail out here with an error
##  instead of trying to patch things up?
${DOEXPIRE} || {
    EXPDIR=
    RMFILE=
}

test -n "${EXPDIR}" && {
    test -z "${REASON}" && REASON="Expiring $$ on ${EXPDIR}"
    EXPIREFLAGS="${EXPIREFLAGS} '-d${EXPDIR}' '-r${REASON}'"
}

test -n "${RMFILE}" && {
    if ${DOGROUPBASEEXPIRY}; then
        EXPIREOVERFLAGS="${EXPIREOVERFLAGS} -z${RMFILE}"
    else
        EXPIREFLAGS="${EXPIREFLAGS} -z${RMFILE}"
    fi
}

test -n "${LOWMARKFILE}" && {
    EXPIREOVERFLAGS="${EXPIREOVERFLAGS} -Z${LOWMARKFILE}"
    if [ "${DOEXPIREOVER}" != "true" ]; then
        echo "warning: lowmark requires expireover"
    fi
}

if ${DOMAIL}; then
    ##  Try to get a temporary file.
    TEMP=${TMPDIR}/doex$$
    test -f ${TEMP} && {
        echo "Temporary file ${TEMP} exists" | eval ${MAIL}
        exit 1
    }
    touch ${TEMP}
    chmod 0660 ${TEMP}
    exec 3>&1 >${TEMP} 2>&1
fi

if [ "${DOPROCBATCH}" = "true" ]; then
    for DIRECTORY in ${PROCBATCHDIR}; do
        if [ -d "${DIRECTORY}" ]; then
            cd "${DIRECTORY}"

            # Find the PIDs of innfeed.
            INNFEEDLOCKS=$(cat *.lock 2>/dev/null | ${SORT} -u)

            # Process files like "innfeed-dropped.A001703" where
            # 1703 is not in $INNFEEDLOCKS.
            for file in $(find . -name "innfeed-dropped*" -print); do
                PID=$(echo "$file" | ${SED} 's/^.*[A-Z]0*//')
                FOUND=false
                for INNFEEDLOCK in $INNFEEDLOCKS; do
                    if [ "$PID" = "$INNFEEDLOCK" ]; then
                        FOUND=true
                    fi
                done
                if [ "$FOUND" = "false" ]; then
                    ${PATHBIN}/procbatch -qmu -t ${DIRECTORY} $file
                fi
            done
        fi
    done
fi

cd ${PATHETC}

##  Show the status of the news system.
${DOSTAT} && {
    ${INNSTAT}
    echo ''
}

##  Lock out others.
trap 'rm -f ${LOCK} ; exit 1' 1 2 3 15
shlock -p $$ -f ${LOCK} || {
    (
        echo "$0: Locked by $(cat ${LOCK})"
        ${INNSTAT}
    ) | eval ${MAIL}
    exit 1
}

##  Run any user programs.
if [ -n "${PROGRAMS}" ]; then
    for P in ${PROGRAMS}; do
        if ${DOMAIL}; then
            echo ''
            echo "${P}:"
        fi
        eval ${P}
    done
fi

${DOLOGS} && {
    if ${DOMAIL}; then
        echo ''
    fi
    scanlogs ${SCANARG}
}

##  Group-based expiry (delete expired overview first).
if [ ${DOGROUPBASEEXPIRY} = true -a ${DOEXPIREOVER} = true ]; then
    if ${DOEXPLOG}; then
        echo "${EXPIREOVER} start $(date): (${EXPIREOVERFLAGS})" >>${EXPLOG}
    fi
    (
        cd ${HISTDIR}
        exec 2>&1
        eval ${EXPIREOVER} "${EXPIREOVERFLAGS}"
    ) \
        | ${SED} -e '/No such file or directory/d' \
            -e 's/^/    /' >>${EXPLOG}
    if ${DOEXPLOG}; then
        echo "${EXPIREOVER} end $(date)" >>${EXPLOG}
    fi
    test -n "${LOWMARKFILE}" && {
        echo "lowmarkrenumber begin $(date): (${LOWMARKFILE})" >>${EXPLOG}
        ctlinnd -s -t$(wc -l <${ACTIVE}) lowmark ${LOWMARKFILE} 2>&1
        echo "lowmarkrenumber end $(date)" >>${EXPLOG}
        rm -f ${MOST_LOGS}/expire.lastlowmark
        mv ${LOWMARKFILE} ${MOST_LOGS}/expire.lastlowmark
    }
    test -n "${RMFILE}" -a -s "${RMFILE}" && {
        mv ${RMFILE} ${RMFILE}.$$ && RMFILE=${RMFILE}.$$

        test -n "${TMPDIR}" && SORT="${SORT} -T ${TMPDIR}"
        ${SORT} -u -o ${RMFILE} ${RMFILE}
        if ${DOEXPLOG}; then
            echo "expirerm start $(date)" >>${EXPLOG}
        fi
        expirerm ${RMFILE}
        if ${DOEXPLOG}; then
            echo "expirerm end $(date)" >>${EXPLOG}
        fi
    }
    DOEXPIREOVER=false
fi

##  The heart of the matter: prologs, expire, epilogs.
if ${DOEXPIRE}; then

    ##  Wait to be fairly certain innwatch is not in the middle of a pass.
    ##  Since we're locked, innwatch will pause now till we're done.
    sleep 30

    ##  See if we're throttled for lack of space.
    SERVERMODE=$(ctlinnd -t $TOUT mode 2>/dev/null | ${SED} 1q)
    case "${SERVERMODE}" in
    'Server paused'*'[innwatch:'*)
        ##  If paused, by innwatch, then turn pause into throttle
        ##  as we're going to stay that way for a while now.
        ctlinnd -t $TOUT -s throttle \
            "$(expr \"${SERVERMODE}\" : 'Server paused \(.*\)')" || {
            (
                echo "$0: Cannot throttle while innwatch paused"
                ${INNSTAT}
            ) | eval ${MAIL}
            exit 1
        }
        ;;
    esac
    case "${SERVERMODE}" in
    "Server throttled"*)
        echo "${SERVERMODE} -- trying to recover"
        THROTTLED=true
        EXPIREFLAGS="${EXPIREFLAGS} -n"
        MESSAGE=
        ;;
    *"[innwatch:"*)
        echo "${SERVERMODE} -- pressing on"
        THROTTLED=false
        EXPIREFLAGS="${EXPIREFLAGS} -n"
        MESSAGE=
        DORENUMBER=false
        ;;
    *)
        THROTTLED=false
        ;;
    esac

    ##  Throttle server if we need to.
    if [ -n "${MESSAGE}" ]; then
        ctlinnd -t $TOUT -s -t120 throttle "${MESSAGE}" 2>&1 || {
            (
                echo "$0: Cannot throttle news"
                ${INNSTAT}
            ) | eval ${MAIL}
            exit 1
        }
    fi

    ## Get rid of an old expire RMFILE since news.daily locks itself and
    ## we would not get here if another instance were still running.
    if [ -n "${RMFILE}" ]; then
        rm -f ${RMFILE}
    fi

    ##  Actual expire the articles (finally!).
    test -n "${EXPIRECTL}" && EXPIREFLAGS="${EXPIREFLAGS} ${EXPIRECTL}"
    if ${DOEXPLOG}; then
        echo "expire begin $(date): (${EXPIREFLAGS})" >>${EXPLOG}
        (
            cd ${HISTDIR}
            exec 2>&1
            eval expire "${EXPIREFLAGS}"
        ) \
            | ${SED} -e '/No such file or directory/d' \
                -e 's/^/    /' >>${EXPLOG}
        echo "expire end $(date)" >>${EXPLOG}
    else
        eval expire "${EXPIREFLAGS}" 2>&1 | grep -v 'No such file or directory'
    fi

    ##  If built on another filesystem, move history files.
    if [ -n "${EXPDIR}" ]; then
        if [ ! -f ${EXPDIR}/history.n -o ! -f ${EXPDIR}/history.n.done ]; then
            (
                echo "$0: No new history files"
                ${INNSTAT}
            ) | eval ${MAIL}
            ##  Remove the possible reserve mode (in case expire was killed).
            if (ctlinnd mode | grep "Reserved ${REASON}" >/dev/null); then
                ctlinnd -s reserve "" 2>/dev/null
            fi
            exit 1
        fi
        cp /dev/null ${HISTORY}
        mv -f ${EXPDIR}/history.n ${HISTORY}
        mv -f ${EXPDIR}/history.n.dir ${HISTORY}.dir
        if [ X${TAGGEDHASH} = XDO ]; then
            mv -f ${EXPDIR}/history.n.pag ${HISTORY}.pag
        else
            mv -f ${EXPDIR}/history.n.index ${HISTORY}.index
            mv -f ${EXPDIR}/history.n.hash ${HISTORY}.hash
        fi
        rm -f ${EXPDIR}/history.n.done

        case "${EXPIREFLAGS}" in
        *-n*) ;;
        *)
            MESSAGE="${REASON}"
            ;;
        esac
    fi

    ##  Restart the server if we need to.
    if ${THROTTLED} || test -n "${MESSAGE}"; then
        ctlinnd -t "$TOUT" -s go "${MESSAGE}" 2>&1 || {
            (
                echo "$0: Cannot unthrottle news"
                ${INNSTAT}
            ) | eval ${MAIL}
            ##  Remove the possible reserve mode (in case expire was killed).
            if (ctlinnd mode | grep "Reserved ${REASON}" >/dev/null); then
                ctlinnd -s reserve "" 2>/dev/null
            fi
            exit 1
        }
    fi
    if ${DOEXPLOG}; then
        echo "all done $(date)" >>${EXPLOG}
    fi
fi

##  Remove old sockets.
${DORM} \
    && find ${TEMPSOCKDIR} -name "${TEMPSOCK}" -mtime +2 -exec rm -f '{}' ';'

##  Remove the possible reserve mode (in case expire was killed).
if (ctlinnd mode | grep "Reserved ${REASON}" >/dev/null); then
    ctlinnd -s reserve "" 2>/dev/null
fi

##  Did we become throttled during the run?
SERVERMODE=$(ctlinnd mode 2>/dev/null | ${SED} 1q)
case "${SERVERMODE}" in
"Server throttled"*)
    ##  We did, try to unthrottle the server.
    echo "${SERVERMODE} -- trying to recover"
    ctlinnd -s go ""
    ;;
esac

##  Release the lock now, everything below this must be able to withstand
##  simultaneous news.daily's running.  We do this so innwatch can start
##  monitoring again asap after the expire is done -- removing the
##  articles isn't critical, nor is the renumber.
rm ${LOCK}

##  Remove the articles that are supposed to go.
if [ ${DOEXPIRE} = true -a ${DOGROUPBASEEXPIRY} != true ]; then
    test -n "${RMFILE}" -a -s "${RMFILE}" && {
        mv ${RMFILE} ${RMFILE}.$$ && RMFILE=${RMFILE}.$$

        test -n "${TMPDIR}" && SORT="${SORT} -T ${TMPDIR}"
        ${SORT} -u -o ${RMFILE} ${RMFILE}
        if ${DOEXPLOG}; then
            echo "expirerm start $(date)" >>${EXPLOG}
        fi
        expirerm ${RMFILE}
        if ${DOEXPLOG}; then
            echo "expirerm end $(date)" >>${EXPLOG}
        fi
    }
fi

##  Remove the articles that are supposed to go from the overview.
if ${DOEXPIREOVER}; then
    if ${DOEXPLOG}; then
        echo "${EXPIREOVER} start $(date)" >>${EXPLOG}
    fi
    (
        cd ${HISTDIR}
        eval ${EXPIREOVER} "${EXPIREOVERFLAGS}" 2>&1
    ) \
        | grep -v 'No such file or directory'
    if ${DOEXPLOG}; then
        echo "${EXPIREOVER} end $(date)" >>${EXPLOG}
    fi
    test -n "${LOWMARKFILE}" && {
        echo "lowmarkrenumber begin $(date): (${LOWMARKFILE})" >>${EXPLOG}
        ctlinnd -s -t$(wc -l <${ACTIVE}) lowmark ${LOWMARKFILE} 2>&1
        echo "lowmarkrenumber end $(date)" >>${EXPLOG}
        rm -f ${MOST_LOGS}/expire.lastlowmark
        mv ${LOWMARKFILE} ${MOST_LOGS}/expire.lastlowmark
    }
fi

##  Renumber the active file.
if ${DORENUMBER}; then
    if ${DOMAIL}; then
        echo ''
        echo 'Renumbering active file.'
    fi
    if [ -z "${RENUMBER}" ]; then
        ctlinnd -s -t$(wc -l <${ACTIVE}) renumber '' 2>&1
    else
        while read GROUP hi lo flag; do
            ctlinnd -s renumber ${GROUP} 2>&1
            sleep ${RENUMBER}
        done <${ACTIVE}
    fi
fi

##  Display expire log messages.
if ${DOMAIL}; then
    if [ -s ${EXPLOG} ]; then
        echo 'Expire messages:'
        cat ${EXPLOG}
        echo '---------'
        echo ''
    fi
fi

##  Show the status of the news system after expiration.
${DOSTAT} && {
    echo 'Post expiration status:'
    echo ''
    ${INNSTAT}
    echo ''
}

##  Run any user programs.
if [ -n "${POSTPROGRAMS}" ]; then
    for P in ${POSTPROGRAMS}; do
        if ${DOMAIL}; then
            echo ''
            echo "${P}:"
        fi
        eval ${P}
    done
fi

##  Mail the report.
if ${DOMAIL}; then
    ##  Stop using the temp file, and mail captured output.
    exec 1>&3 2>&1 3>&-
    MAIL="${MAILCMD} -a '${ADDHEADER}' -s '${MAILSUBJ}' ${NEWSMASTER}"
    test -s ${TEMP} && cat ${TEMP} | ${SED} -e 's/^~/~~/' | eval ${MAIL}
    rm -f ${TEMP}
fi

##  All done
if ${DAILY}; then
    date >${PATHDB}/.news.daily
fi
${RNEWS} -U
exit 0
