#!/bin/bash
# +------------------------------------------------------------------+
# |             ____ _               _        __  __ _  __           |
# |            / ___| |__   ___  ___| | __   |  \/  | |/ /           |
# |           | |   | '_ \ / _ \/ __| |/ /   | |\/| | ' /            |
# |           | |___| | | |  __/ (__|   <    | |  | | . \            |
# |            \____|_| |_|\___|\___|_|\_\___|_|  |_|_|\_\           |
# |                                                                  |
# | Copyright Mathias Kettner 2014             mk@mathias-kettner.de |
# +------------------------------------------------------------------+
#
# This file is part of Check_MK.
# The official homepage is at http://mathias-kettner.de/check_mk.
#
# check_mk is free software;  you can redistribute it and/or modify it
# under the  terms of the  GNU General Public License  as published by
# the Free Software Foundation in version 2.  check_mk is  distributed
# in the hope that it will be useful, but WITHOUT ANY WARRANTY;  with-
# out even the implied warranty of  MERCHANTABILITY  or  FITNESS FOR A
# PARTICULAR PURPOSE. See the  GNU General Public License for more de-
# tails. You should have  received  a copy of the  GNU  General Public
# License along with GNU Make; see the file  COPYING.  If  not,  write
# to the Free Software Foundation, Inc., 51 Franklin St,  Fifth Floor,
# Boston, MA 02110-1301 USA.

# Check_MK agent plugin for monitoring ORACLE databases
# This plugin is a result of the common work of Thorsten Bruhns
# and Mathias Kettner. Thorsten is responsible for the ORACLE
# stuff, Mathias for the shell hacking...
# This plugin is available for linux AND solaris.

# Example for mk_oracle.cfg
# DBUSER=<DBUSER>:<PASSWORD>:<SYSDBA>:<HOSTNAME>:<Port>
# ASMUSER=<DBUSER>:<PASSWORD>:<SYSDBA/SYSASM>:<HOSTNAME>:<Port>
#
# SYSDBA or SYSASM is optional but needed for a mounted instance
# HOSTNAME is optional - Default is localhost
# PORT is optional - Default is 1521

# ONLY_SIDS is only usable for local running instances. It is ignored for
# REMOTE_-Instances. The same applies to EXCLUDE.

while test $# -gt 0
do
    if [ "${1}" = '-d' ] ; then
        set -xv ; DEBUG=1
    elif [ "${1}" = '-t' ] ; then
        DEBUGCONNECT=1
    fi
    shift
done

if [ ! "$MK_CONFDIR" ] ; then
    echo "MK_CONFDIR not set!" >&2
    exit 1
fi

if [ ! "$MK_VARDIR" ] ; then
    export MK_VARDIR=$MK_CONFDIR
fi


#   .--Config--------------------------------------------------------------.
#   |                     ____             __ _                            |
#   |                    / ___|___  _ __  / _(_) __ _                      |
#   |                   | |   / _ \| '_ \| |_| |/ _` |                     |
#   |                   | |__| (_) | | | |  _| | (_| |                     |
#   |                    \____\___/|_| |_|_| |_|\__, |                     |
#   |                                           |___/                      |
#   +----------------------------------------------------------------------+
#   | The user can override and set variables in mk_oracle.cfg             |
#   '----------------------------------------------------------------------'

# Sections that run fast and do no caching
SYNC_SECTIONS="instance sessions logswitches undostat recovery_area processes recovery_status longactivesessions dataguard_stats performance locks"

# Sections that are run in the background and at a larger interval.
# Note: sections not listed in SYNC_SECTIONS or ASYNC_SECTIONS will not be
# executed at all!
ASYNC_SECTIONS="tablespaces rman jobs ts_quotas resumable"

# Sections that are run in the background and at a larger interval.
# Note: _ASM_ sections are only executed when SID starts with '+'
#       sections listed in SYNC_SECTIONS or ASYNC_SECTIONS are not
#       executed for ASM.
SYNC_ASM_SECTIONS="instance processes"
ASYNC_ASM_SECTIONS="asm_diskgroup"

# Interval for running async checks (in seconds)
CACHE_MAXAGE=600

# You can specify a list of SIDs to monitor. Those databases will
# only be handled, if they are found running, though!
#
#   ONLY_SIDS="XE ORCL FOO BAR"
#
# It is possible to filter SIDS negatively. Just add the following to
# the mk_oracle.cfg file:
#
#   EXCLUDE_<sid>="ALL"
#
# Another option is to filter single checks for SIDS. Just add
# lines as follows to the mk_oracle.cfg file. One service per
# line:
#
#   EXCLUDE_<sid>="<service>"
#
# For example skip oracle_sessions and oracle_logswitches checks
# for the instance "mysid".
#
#   EXCLUDE_mysid="sessions logswitches"
#

# Source the optional configuration file for this agent plugin
if [ -e "$MK_CONFDIR/mk_oracle.cfg" ]
then
    . $MK_CONFDIR/mk_oracle.cfg
fi

#.
#   .--SQL Queries---------------------------------------------------------.
#   |        ____   ___  _        ___                  _                   |
#   |       / ___| / _ \| |      / _ \ _   _  ___ _ __(_) ___  ___         |
#   |       \___ \| | | | |     | | | | | | |/ _ \ '__| |/ _ \/ __|        |
#   |        ___) | |_| | |___  | |_| | |_| |  __/ |  | |  __/\__ \        |
#   |       |____/ \__\_\_____|  \__\_\\__,_|\___|_|  |_|\___||___/        |
#   |                                                                      |
#   +----------------------------------------------------------------------+
#   | The following functions create SQL queries for ORACLE and output     |
#   | them to stdout. All queries output the database name or the instane  |
#   | name as first column.                                                |
#   '----------------------------------------------------------------------'

sql_performance()
{
    if [ "$AT_LEAST_ORACLE_101" = 'yes' ] ; then
        echo 'PROMPT <<<oracle_performance:sep(124)>>>'
        echo "select upper(i.INSTANCE_NAME)
                     ||'|'|| 'sys_time_model'
                     ||'|'|| S.STAT_NAME
                     ||'|'|| Round(s.value/1000000)
              from v\$instance i,
                   v\$sys_time_model s
              where s.stat_name in('DB time', 'DB CPU')
              order by s.stat_name;
              select upper(i.INSTANCE_NAME)
                     ||'|'|| 'buffer_pool_statistics'
                     ||'|'|| b.name
                     ||'|'|| b.db_block_gets
                     ||'|'|| b.db_block_change
                     ||'|'|| b.consistent_gets
                     ||'|'|| b.physical_reads
                     ||'|'|| b.physical_writes
                     ||'|'|| b.FREE_BUFFER_WAIT
                     ||'|'|| b.BUFFER_BUSY_WAIT
              from v\$instance i, V\$BUFFER_POOL_STATISTICS b;
              select upper(i.INSTANCE_NAME)
                     ||'|'|| 'SGA_info'
                     ||'|'|| s.name
                     ||'|'|| s.bytes
              from v\$sgainfo s, v\$instance i;
              select upper(i.INSTANCE_NAME)
                     ||'|'|| 'librarycache'
                     ||'|'|| b.namespace
                     ||'|'|| b.gets
                     ||'|'|| b.gethits
                     ||'|'|| b.pins
                     ||'|'|| b.pinhits
                     ||'|'|| b.reloads
                     ||'|'|| b.invalidations
              from v\$instance i, V\$librarycache b;"
    fi
}

sql_tablespaces()
{
    echo 'PROMPT <<<oracle_tablespaces:sep(124)>>>'
    if [ "$AT_LEAST_ORACLE_102" = 'yes' ] ; then

        echo "select upper(decode(${IGNORE_DB_NAME:-0}, 0, d.NAME, i.instance_name))
                  || '|' || file_name ||'|'|| tablespace_name ||'|'|| fstatus ||'|'|| AUTOEXTENSIBLE
                  ||'|'|| blocks ||'|'|| maxblocks ||'|'|| USER_BLOCKS ||'|'|| INCREMENT_BY
                  ||'|'|| ONLINE_STATUS ||'|'|| BLOCK_SIZE
                  ||'|'|| decode(tstatus,'READ ONLY', 'READONLY', tstatus) || '|' || free_blocks
                  ||'|'|| contents
                  ||'|'|| iversion
           from v\$database d , v\$instance i, (
                    select f.file_name, f.tablespace_name, f.status fstatus, f.AUTOEXTENSIBLE,
                    f.blocks, f.maxblocks, f.USER_BLOCKS, f.INCREMENT_BY,
                    f.ONLINE_STATUS, t.BLOCK_SIZE, t.status tstatus, nvl(sum(fs.blocks),0) free_blocks, t.contents,
                    (select version from v\$instance) iversion
                    from dba_data_files f, dba_tablespaces t, dba_free_space fs
                    where f.tablespace_name = t.tablespace_name
                    and f.file_id = fs.file_id(+)
                    group by f.file_name, f.tablespace_name, f.status, f.autoextensible,
                    f.blocks, f.maxblocks, f.user_blocks, f.increment_by, f.online_status,
                    t.block_size, t.status, t.contents)
           where d.database_role = 'PRIMARY';

           select upper(decode(${IGNORE_DB_NAME:-0}
                                 , 0, dbf.name
                                 , i.instance_name))
                  || '|' || dbf.file_name
                  || '|' || dbf.tablespace_name
                  || '|' || dbf.fstatus
                  || '|' || dbf.AUTOEXTENSIBLE
                  || '|' || dbf.blocks
                  || '|' || dbf.maxblocks
                  || '|' || dbf.USER_BLOCKS
                  || '|' || dbf.INCREMENT_BY
                  || '|' || dbf.ONLINE_STATUS
                  || '|' || dbf.BLOCK_SIZE
                  || '|' || decode(tstatus,'READ ONLY', 'READONLY', tstatus)
                  || '|' || dbf.free_blocks
                  || '|' || 'TEMPORARY'
                  || '|' || i.version
           FROM v\$database d
           JOIN v\$instance i ON 1 = 1
           JOIN (
                 SELECT vp.name,
                        f.file_name,
                        t.tablespace_name,
                        f.status fstatus,
                        f.autoextensible,
                        f.blocks,
                        f.maxblocks,
                        f.user_blocks,
                        f.increment_by,
                        'ONLINE' online_status,
                        t.block_size,
                        t.status tstatus,
                        f.blocks - nvl(SUM(tu.blocks),0) free_blocks,
                        t.contents
                 FROM dba_tablespaces t
                 JOIN ( SELECT 0
                             ,name
                       FROM v\$database
                      ) vp ON 1=1
                 LEFT OUTER JOIN dba_temp_files f ON t.tablespace_name = f.tablespace_name
                 LEFT OUTER JOIN gv\$tempseg_usage tu ON f.tablespace_name = tu.tablespace
                                                      AND f.RELATIVE_FNO = tu.SEGRFNO#
                 WHERE t.contents = 'TEMPORARY'
                 GROUP BY vp.name,
                          f.file_name,
                          t.tablespace_name,
                          f.status,
                          f.autoextensible,
                          f.blocks,
                          f.maxblocks,
                          f.user_blocks,
                          f.increment_by,
                          t.block_size,
                          t.status,
                          t.contents
                ) dbf ON 1 = 1;"

    elif [ "$AT_LEAST_ORACLE_92" = 'yes' ] ; then

        echo "select upper(decode(${IGNORE_DB_NAME:-0}, 0, d.NAME, i.instance_name))
                  || '|' || file_name ||'|'|| tablespace_name ||'|'|| fstatus ||'|'|| AUTOEXTENSIBLE
                  ||'|'|| blocks ||'|'|| maxblocks ||'|'|| USER_BLOCKS ||'|'|| INCREMENT_BY
                  ||'|'|| ONLINE_STATUS ||'|'|| BLOCK_SIZE
                  ||'|'|| decode(tstatus,'READ ONLY', 'READONLY', tstatus) || '|' || free_blocks
                  ||'|'|| contents
           from v\$database d , v\$instance i, (
                    select f.file_name, f.tablespace_name, f.status fstatus, f.AUTOEXTENSIBLE,
                    f.blocks, f.maxblocks, f.USER_BLOCKS, f.INCREMENT_BY,
                    'ONLINE' ONLINE_STATUS, t.BLOCK_SIZE, t.status tstatus, nvl(sum(fs.blocks),0) free_blocks, t.contents
                    from dba_data_files f, dba_tablespaces t, dba_free_space fs
                    where f.tablespace_name = t.tablespace_name
                    and f.file_id = fs.file_id(+)
                    group by f.file_name, f.tablespace_name, f.status, f.autoextensible,
                    f.blocks, f.maxblocks, f.user_blocks, f.increment_by, 'ONLINE',
                    t.block_size, t.status, t.contents
                    UNION
                    select f.file_name, f.tablespace_name, 'ONLINE' status, f.AUTOEXTENSIBLE,
                    f.blocks, f.maxblocks, f.USER_BLOCKS, f.INCREMENT_BY, 'TEMP',
                    t.BLOCK_SIZE, 'TEMP' status, sum(sh.blocks_free) free_blocks, 'TEMPORARY'
                    from v\$thread th, dba_temp_files f, dba_tablespaces t, v\$temp_space_header sh
                    WHERE f.tablespace_name = t.tablespace_name and f.file_id = sh.file_id
                    GROUP BY th.instance, f.file_name, f.tablespace_name, 'ONLINE',
                    f.autoextensible, f.blocks, f.maxblocks, f.user_blocks, f.increment_by,
                    'TEMP', t.block_size, t.status);
              "
    fi
}

sql_dataguard_stats()
{
    if [ "$AT_LEAST_ORACLE_102" = 'yes' ] ; then
        echo 'PROMPT <<<oracle_dataguard_stats:sep(124)>>>'
        echo "SELECT upper(decode(${IGNORE_DB_NAME:-0}, 0, d.NAME, i.instance_name))
                     ||'|'|| upper(d.DB_UNIQUE_NAME)
                     ||'|'|| d.DATABASE_ROLE
                     ||'|'|| ds.name
                     ||'|'|| ds.value
                     ||'|'|| d.SWITCHOVER_STATUS
              FROM  v\$database d
              JOIN  v\$parameter vp on 1=1
              JOIN v\$instance i on 1=1
              left outer join V\$dataguard_stats ds on 1=1
              WHERE vp.name = 'log_archive_config'
              AND   vp.value is not null
              ORDER BY 1;
             "
    fi
}

sql_recovery_status()
{
    echo 'PROMPT <<<oracle_recovery_status:sep(124)>>>'
    if [ "$AT_LEAST_ORACLE_101" = 'yes' ] ; then
        echo "SELECT upper(decode(${IGNORE_DB_NAME:-0}, 0, d.NAME, i.instance_name))
                     ||'|'|| d.DB_UNIQUE_NAME
                     ||'|'|| d.DATABASE_ROLE
                     ||'|'|| d.open_mode
                     ||'|'|| dh.file#
                     ||'|'|| round((dh.CHECKPOINT_TIME-to_date('01.01.1970','dd.mm.yyyy'))*24*60*60)
                     ||'|'|| round((sysdate-dh.CHECKPOINT_TIME)*24*60*60)
                     ||'|'|| dh.STATUS
                     ||'|'|| dh.RECOVER
                     ||'|'|| dh.FUZZY
                     ||'|'|| dh.CHECKPOINT_CHANGE#
              FROM  V\$datafile_header dh, v\$database d, v\$instance i
              ORDER BY dh.file#;
             "
    elif [ "$AT_LEAST_ORACLE_92" = 'yes' ] ; then
        echo "SELECT upper(decode(${IGNORE_DB_NAME:-0}, 0, d.NAME, i.instance_name))
                     ||'|'|| upper(decode(${IGNORE_DB_NAME:-0}, 0, d.NAME, i.instance_name))
                     ||'|'|| d.DATABASE_ROLE
                     ||'|'|| d.open_mode
                     ||'|'|| dh.file#
                     ||'|'|| round((dh.CHECKPOINT_TIME-to_date('01.01.1970','dd.mm.yyyy'))*24*60*60)
                     ||'|'|| round((sysdate-dh.CHECKPOINT_TIME)*24*60*60)
                     ||'|'|| dh.STATUS
                     ||'|'|| dh.RECOVER
                     ||'|'|| dh.FUZZY
                     ||'|'|| dh.CHECKPOINT_CHANGE#
              FROM  V\$datafile_header dh, v\$database d, v\$instance i
              ORDER BY dh.file#;
             "
    fi
}

sql_rman()
{
    if [ "$AT_LEAST_ORACLE_102" = 'yes' ] ; then
        echo 'PROMPT <<<oracle_rman:sep(124)>>>'
        echo "select /*"$HINT_RMAN" check_mk rman1 */ upper(name)
                     || '|'|| 'COMPLETED'
                     || '|'|| to_char(COMPLETION_TIME, 'YYYY-mm-dd_HH24:MI:SS')
                     || '|'|| to_char(COMPLETION_TIME, 'YYYY-mm-dd_HH24:MI:SS')
                     || '|'|| case when INCREMENTAL_LEVEL IS NULL
                              then 'DB_FULL'
                              else 'DB_INCR'
                              end
                     || '|'|| INCREMENTAL_LEVEL
                     || '|'|| round(((sysdate-COMPLETION_TIME) * 24 * 60), 0)
                     || '|'|| INCREMENTAL_CHANGE#
                from (select upper(decode(${IGNORE_DB_NAME:-0}, 0, vd.NAME, i.instance_name)) name
                           , bd2.INCREMENTAL_LEVEL, bd2.INCREMENTAL_CHANGE#, min(bd2.COMPLETION_TIME) COMPLETION_TIME
                      from (select bd.file#, bd.INCREMENTAL_LEVEL, max(bd.COMPLETION_TIME) COMPLETION_TIME
                            from v\$backup_datafile bd
                            join v\$datafile_header dh on dh.file# = bd.file#
                            where dh.status = 'ONLINE'
                            group by bd.file#, bd.INCREMENTAL_LEVEL
                                           ) bd
                     join v\$backup_datafile bd2 on bd2.file# = bd.file#
                                               and bd2.COMPLETION_TIME = bd.COMPLETION_TIME
                     join v\$database vd on vd.RESETLOGS_CHANGE# = bd2.RESETLOGS_CHANGE#
                     join v\$instance i on 1=1
                     group by upper(decode(${IGNORE_DB_NAME:-0}, 0, vd.NAME, i.instance_name))
                            , bd2.INCREMENTAL_LEVEL
                            , bd2.INCREMENTAL_CHANGE#
                     order by name, bd2.INCREMENTAL_LEVEL);

              select /*"$HINT_RMAN" check_mk rman2 */ name
                    || '|' || 'COMPLETED'
                    || '|'
                    || '|' || to_char(CHECKPOINT_TIME, 'yyyy-mm-dd_hh24:mi:ss')
                    || '|' || 'CONTROLFILE'
                    || '|'
                    || '|' || round((sysdate - CHECKPOINT_TIME) * 24 * 60)
                    || '|' || '0'
              from (select upper(decode(${IGNORE_DB_NAME:-0}, 0, d.NAME, i.instance_name)) name
                          ,max(bcd.CHECKPOINT_TIME) CHECKPOINT_TIME
                    from v\$database d
                    join V\$BACKUP_CONTROLFILE_DETAILS bcd on d.RESETLOGS_CHANGE# = bcd.RESETLOGS_CHANGE#
                    join v\$instance i on 1=1
                    group by upper(decode(${IGNORE_DB_NAME:-0}, 0, d.NAME, i.instance_name))
                   );

              select /*"$HINT_RMAN" check_mk rman3 */ name
                     || '|COMPLETED'
                     || '|'|| to_char(sysdate, 'YYYY-mm-dd_HH24:MI:SS')
                     || '|'|| to_char(completed, 'YYYY-mm-dd_HH24:MI:SS')
                     || '|ARCHIVELOG||'
                     || round((sysdate - completed)*24*60,0)
                     || '|'
              from (
                    select upper(decode(${IGNORE_DB_NAME:-0}, 0, d.NAME, i.instance_name)) name
                         , max(a.completion_time) completed
                         , case when a.backup_count > 0 then 1 else 0 end
                    from v\$archived_log a, v\$database d, v\$instance i
                    where a.backup_count > 0
                          and a.dest_id in
                          (select b.dest_id
                           from v\$archive_dest b
                           where b.target = 'PRIMARY'
                             and b.SCHEDULE = 'ACTIVE'
                          )
                    group by d.NAME, i.instance_name
                           , case when a.backup_count > 0 then 1 else 0 end);"
    fi
}

sql_recovery_area()
{
    if [ "$AT_LEAST_ORACLE_102" = 'yes' ] ; then
        echo 'PROMPT <<<oracle_recovery_area>>>'
        echo "select upper(decode(${IGNORE_DB_NAME:-0}, 0, d.NAME, i.instance_name))
                     ||' '|| round((SPACE_USED-SPACE_RECLAIMABLE)/
                               (CASE NVL(SPACE_LIMIT,1) WHEN 0 THEN 1 ELSE SPACE_LIMIT END)*100)
                     ||' '|| round(SPACE_LIMIT/1024/1024)
                     ||' '|| round(SPACE_USED/1024/1024)
                     ||' '|| round(SPACE_RECLAIMABLE/1024/1024)
                     ||' '|| d.FLASHBACK_ON
              from V\$RECOVERY_FILE_DEST, v\$database d, v\$instance i;
             "
    fi
}

sql_undostat()
{
    echo 'PROMPT <<<oracle_undostat:sep(124)>>>'
    if [ "$AT_LEAST_ORACLE_102" = 'yes' ] ; then
        echo "select upper(i.INSTANCE_NAME)
                     ||'|'|| ACTIVEBLKS
                     ||'|'|| MAXCONCURRENCY
                     ||'|'|| TUNED_UNDORETENTION
                     ||'|'|| maxquerylen
                     ||'|'|| NOSPACEERRCNT
              from v\$instance i,
                  (select * from (select *
                                  from v\$undostat order by end_time desc
                                 )
                            where rownum = 1
                              and TUNED_UNDORETENTION > 0
                  );
             "
    elif [ "$AT_LEAST_ORACLE_92" = 'yes' ] ; then
        # TUNED_UNDORETENTION and ACTIVEBLKS are not availibe in Oracle <=9.2!
        # we sent a -1 for filtering in check_undostat
        echo "select upper(i.INSTANCE_NAME)
                     ||'|-1'
                     ||'|'|| MAXCONCURRENCY
                     ||'|-1'
                     ||'|'|| maxquerylen
                     ||'|'|| NOSPACEERRCNT
                  from v\$instance i,
                  (select * from (select *
                                  from v\$undostat order by end_time desc
                                 )
                            where rownum = 1
                  );
             "
    fi
}

sql_resumable()
{
    echo 'PROMPT <<<oracle_resumable:sep(124)>>>'
    echo "select upper(i.INSTANCE_NAME)
                 ||'|'|| u.username
                 ||'|'|| a.SESSION_ID
                 ||'|'|| a.status
                 ||'|'|| a.TIMEOUT
                 ||'|'|| round((sysdate-to_date(a.SUSPEND_TIME,'mm/dd/yy hh24:mi:ss'))*24*60*60)
                 ||'|'|| a.ERROR_NUMBER
                 ||'|'|| to_char(to_date(a.SUSPEND_TIME, 'mm/dd/yy hh24:mi:ss'),'mm/dd/yy_hh24:mi:ss')
                 ||'|'|| a.RESUME_TIME
                 ||'|'|| a.ERROR_MSG
          from dba_resumable a, v\$instance i, dba_users u
          where a.INSTANCE_ID = i.INSTANCE_NUMBER
          and u.user_id = a.user_id
          and a.SUSPEND_TIME is not null
          union all
          select upper(i.INSTANCE_NAME)
                 || '|||||||||'
          from v\$instance i
;
         "
}

sql_jobs()
{
    if [ "$AT_LEAST_ORACLE_102" = 'yes' ] ; then
        echo 'PROMPT <<<oracle_jobs:sep(124)>>>'
        echo "SELECT upper(decode(${IGNORE_DB_NAME:-0}, 0, vd.NAME, i.instance_name))
                     ||'|'|| j.OWNER
                     ||'|'|| j.JOB_NAME
                     ||'|'|| j.STATE
                     ||'|'|| ROUND((TRUNC(sysdate) + j.LAST_RUN_DURATION - TRUNC(sysdate)) * 86400)
                     ||'|'|| j.RUN_COUNT
                     ||'|'|| j.ENABLED
                     ||'|'|| NVL(j.NEXT_RUN_DATE, to_date('1970-01-01', 'YYYY-mm-dd'))
                     ||'|'|| NVL(j.SCHEDULE_NAME, '-')
                     ||'|'|| jd.STATUS
              FROM dba_scheduler_jobs j
              join v\$database vd on 1 = 1
              join v\$instance i on 1 = 1
              left outer join (SELECT owner, job_name, max(LOG_ID) log_id
                                        FROM dba_scheduler_job_run_details dd
                                        group by owner, job_name
                              ) jm on  jm.JOB_NAME = j.JOB_NAME
                                   and jm.owner=j.OWNER
              left outer join dba_scheduler_job_run_details jd
                              on  jd.owner = jm.OWNER
                              AND jd.JOB_NAME = jm.JOB_NAME
                              AND jd.LOG_ID = jm.LOG_ID
              WHERE j.auto_drop = 'FALSE';"
    fi
}

sql_ts_quotas()
{
    echo 'PROMPT <<<oracle_ts_quotas:sep(124)>>>'
    echo "select upper(decode(${IGNORE_DB_NAME:-0}, 0, d.NAME, i.instance_name))
                 ||'|'|| Q.USERNAME
                 ||'|'|| Q.TABLESPACE_NAME
                 ||'|'|| Q.BYTES
                 ||'|'|| Q.MAX_BYTES
          from dba_ts_quotas Q, v\$database d, v\$instance i
          where max_bytes > 0
          union all
          select upper(decode(${IGNORE_DB_NAME:-0}, 0, d.NAME, i.instance_name))
                 ||'|||'
          from v\$database d, v\$instance i
          order by 1;
         "
}

sql_version()
{
    echo 'PROMPT <<<oracle_version>>>'
    echo "select upper(i.INSTANCE_NAME)
	  || ' ' || banner
	  from v\$version, v\$instance i
	  where banner like 'Oracle%';"
}

sql_instance()
{
    echo 'prompt <<<oracle_instance:sep(124)>>>'
    if [ ${ORACLE_SID:0:1} = '+' ] ; then
        # ASM
        echo "select upper(i.instance_name)
                     || '|' || i.VERSION
                     || '|' || i.STATUS
                     || '|' || i.LOGINS
                     || '|' || i.ARCHIVER
                     || '|' || round((sysdate - i.startup_time) * 24*60*60)
                     || '|' || '0'
                     || '|' || 'NO'
                     || '|' || 'ASM'
                     || '|' || 'NO'
                     || '|' || i.instance_name
                from v\$instance i;
             "
    else
        # normal Instance
        echo "select upper(i.instance_name)
                     || '|' || i.VERSION
                     || '|' || i.STATUS
                     || '|' || i.LOGINS
                     || '|' || i.ARCHIVER
                     || '|' || round((sysdate - i.startup_time) * 24*60*60)
                     || '|' || DBID
                     || '|' || LOG_MODE
                     || '|' || DATABASE_ROLE
                     || '|' || FORCE_LOGGING
                     || '|' || d.name
                     || '|' || to_char(d.created, 'ddmmyyyyhh24mi')
                from v\$instance i, v\$database d;
             "
    fi
}

sql_sessions()
{
    echo 'prompt <<<oracle_sessions:sep(124)>>>'
    echo "select upper(i.instance_name)
                 || '|' || CURRENT_UTILIZATION
                 || '|' || ltrim(LIMIT_VALUE)
                 || '|' || MAX_UTILIZATION
          from v\$resource_limit, v\$instance i
          where RESOURCE_NAME = 'sessions';"

}

sql_processes()
{
    echo 'prompt <<<oracle_processes>>>'
    echo "select upper(i.instance_name)
                  || ' ' || CURRENT_UTILIZATION
                  || ' ' || ltrim(rtrim(LIMIT_VALUE))
           from v\$resource_limit, v\$instance i
           where RESOURCE_NAME = 'processes';
         "
}

sql_logswitches()
{
    echo 'prompt <<<oracle_logswitches>>>'
    echo "select upper(i.instance_name)
                  || ' ' || logswitches
           from v\$instance i ,
                (select count(1) logswitches
                 from v\$loghist h , v\$instance i
                 where h.first_time > sysdate - 1/24
                 and h.thread# = i.instance_number
                );
         "
}

sql_locks()
{
    if [ "$AT_LEAST_ORACLE_102" = 'yes' ] ; then
        echo 'prompt <<<oracle_locks:sep(124)>>>'
        echo "select upper(i.instance_name)
                     || '|' || b.sid
                     || '|' || b.serial#
                     || '|' || b.machine
                     || '|' || b.program
                     || '|' || b.process
                     || '|' || b.osuser
                     || '|' || b.username
                     || '|' || b.SECONDS_IN_WAIT
                     || '|' || b.BLOCKING_SESSION_STATUS
                     || '|' || bs.inst_id
                     || '|' || bs.sid
                     || '|' || bs.serial#
                     || '|' || bs.machine
                     || '|' || bs.program
                     || '|' || bs.process
                     || '|' || bs.osuser
                     || '|' || bs.username
              from v\$session b
              join v\$instance i on 1=1
              join gv\$session bs on bs.inst_id = b.BLOCKING_INSTANCE
                                 and bs.sid = b.BLOCKING_SESSION
              where b.BLOCKING_SESSION is not null
;
              select upper(i.instance_name)
                     || '|||||||||||||||||'
              from v\$instance i
;
        "
    fi
}

sql_locks_old()
{
    if [ "$AT_LEAST_ORACLE_101" = 'yes' ] ; then
        echo 'prompt <<<oracle_locks:sep(124)>>>'
        echo "SET SERVEROUTPUT ON feedback off
DECLARE
    type x is table of varchar2(20000) index by pls_integer;
    xx x;
begin
    begin
        execute immediate 'select upper(i.instance_name)
           || ''|'' || a.sid
           || ''|'' || b.serial#
           || ''|'' || b.machine
           || ''|'' || b.program
           || ''|'' || b.process
           || ''|'' || b.osuser
           || ''|'' || a.ctime
           || ''|'' || decode(c.owner,NULL,''NULL'',c.owner)
           || ''|'' || decode(c.object_name,NULL,''NULL'',c.object_name)
            from V\$LOCK a, v\$session b, dba_objects c, v\$instance i
            where (a.id1, a.id2, a.type)
                       IN (SELECT id1, id2, type
                           FROM GV\$LOCK
                           WHERE request>0
                          )
            and request=0
            and a.sid = b.sid
            and a.id1 = c.object_id (+)
            union all
            select upper(i.instance_name) || ''|||||||||''
            from  v\$instance i'
        bulk collect into xx;
        if xx.count >= 1 then
            for i in 1 .. xx.count loop
                dbms_output.put_line(xx(i));
            end loop;
        end if;
    exception
        when others then
            for cur1 in (select upper(i.instance_name) instance_name from  v\$instance i) loop
                dbms_output.put_line(cur1.instance_name || '|||||||||'||sqlerrm);
            end loop;
    end;
END;
/
set serverout off
"
    fi
}

sql_longactivesessions()
{
    if [ "$AT_LEAST_ORACLE_101" = 'yes' ] ; then
        echo 'prompt <<<oracle_longactivesessions:sep(124)>>>'
        echo "select upper(i.instance_name)
                     || '|' || s.sid
                     || '|' || s.serial#
                     || '|' || s.machine
                     || '|' || s.process
                     || '|' || s.osuser
                     || '|' || s.program
                     || '|' || s.last_call_et
                     || '|' || s.sql_id
              from v\$session s, v\$instance i
              where s.status = 'ACTIVE'
              and type != 'BACKGROUND'
              and s.username is not null
              and s.username not in('PUBLIC')
              and s.last_call_et > 60*60
              union all
              select upper(i.instance_name)
                     || '||||||||'
              from v\$instance i;
             "
    fi
}

sql_asm_diskgroup()
{
    echo 'prompt <<<oracle_asm_diskgroup>>>'
    if [ "$AT_LEAST_ORACLE_112" = 'yes' ] ; then
        echo "select STATE
                     || ' ' || TYPE
                     || ' ' || 'N'
                     || ' ' || sector_size
                     || ' ' || block_size
                     || ' ' || allocation_unit_size
                     || ' ' || total_mb
                     || ' ' || free_mb
                     || ' ' || required_mirror_free_mb
                     || ' ' || usable_file_mb
                     || ' ' || offline_disks
                     || ' ' || voting_files
                     || ' ' || name || '/'
                from v\$asm_diskgroup;
             "
    elif [ "$AT_LEAST_ORACLE_102" = 'yes' ] ; then
        echo "select STATE
                     || ' ' || TYPE
                     || ' ' || 'N'
                     || ' ' || sector_size
                     || ' ' || block_size
                     || ' ' || allocation_unit_size
                     || ' ' || total_mb
                     || ' ' || free_mb
                     || ' ' || required_mirror_free_mb
                     || ' ' || usable_file_mb
                     || ' ' || offline_disks
                     || ' ' || 'N'
                     || ' ' || name || '/'
                from v\$asm_diskgroup;
             "
    fi
}

#.
#   .--set_osenv-----------------------------------------------------------.
#   |                       _                                              |
#   |              ___  ___| |_     ___  ___  ___ _ ____   __              |
#   |             / __|/ _ \ __|   / _ \/ __|/ _ \ '_ \ \ / /              |
#   |             \__ \  __/ |_   | (_) \__ \  __/ | | \ V /               |
#   |             |___/\___|\__|___\___/|___/\___|_| |_|\_/                |
#   |                         |_____|                                      |
#   +----------------------------------------------------------------------+
#   | Functions for Operating System dependent stuff                       |
#   '----------------------------------------------------------------------'

function set_osenv () {
    ostype=$(uname -s)
    AWK=$(which awk)

    if [ ${ostype} = 'Linux' ] ; then

        GREP=$(which grep)
        STATCX='stat -c %X'
        STATCY='stat -c %Y'

    elif [ ${ostype} = 'SunOS' ] ; then

        # expand the PATH for inetd. Otherwise some stuff in /opt/sfw/bin is not found!
        export PATH=$PATH:/usr/ucb:/usr/proc/bin:opt/sfw/bin:/opt/sfw/sbin:/usr/sfw/bin:/usr/sfw/sbin:/opt/csw/bin

        GREP=/usr/xpg4/bin/grep

        if [ ! -x $GREP ] ; then
            echo "Please make sure that "$GREP" is existing on Solaris!"
            echo "Aborting mk_oracle plugin."
            exit 999
        fi

        STATCX='file_mtime'
        STATCY='file_mtime'

        AWK=$(which nawk)

    else
        ostype="unknown OS: "${ostype}
    fi

}

#.
#   .--oraenv--------------------------------------------------------------.
#   |                                                                      |
#   |                    ___  _ __ __ _  ___ _ ____   __                   |
#   |                   / _ \| '__/ _` |/ _ \ '_ \ \ / /                   |
#   |                  | (_) | | | (_| |  __/ | | \ V /                    |
#   |                   \___/|_|  \__,_|\___|_| |_|\_/                     |
#   |                                                                      |
#   +----------------------------------------------------------------------+
#   | Functions for getting the Oracle environment                         |
#   '----------------------------------------------------------------------'

function set_oraenv () {
    local SID=${1}

    if [[ "$SID" =~ ^REMOTE_INSTANCE_.* ]] ; then

        # we get the ORACLE_HOME from mk_oracle.cfg for REMOTE execution
        ORACLE_HOME=${ORACLE_HOME:-${REMOTE_ORACLE_HOME}}

    else
        # we need to keep an existing ORACLE_SID in remote mode
        # => set it only in local mode
        ORACLE_SID=$SID

        # we work in local mode
        test -f /etc/oratab && ORATAB=/etc/oratab
        # /var/opt/oracle/oratab is needed for Oracle Solaris
        test -f /var/opt/oracle/oratab && ORATAB=/var/opt/oracle/oratab
        test -f ${ORATAB:-""} || echo "ORA-99999 oratab not found"
        test -f ${ORATAB:-""} || exit 1

        ORACLE_HOME=$(cat ${ORATAB} | grep "^"${ORACLE_SID}":" | cut -d":" -f2)
        if [ -z $ORACLE_HOME ] ; then
            # cut last number from SID for Oracle RAC to find entry in oratab
            ORACLE_SID_SHORT=$(echo $ORACLE_SID | sed "s/[0-9]$//")
            ORACLE_HOME=$(cat ${ORATAB} | grep "^"${ORACLE_SID_SHORT}":" | cut -d":" -f2)
        fi
    fi

    LD_LIBRARY_PATH=$ORACLE_HOME/lib

    if [ ! -d ${ORACLE_HOME:-"not_found"} ] ; then
        echo "ORA-99999 ORACLE_HOME for ORACLE_SID="$ORACLE_SID" not found or not existing!"
        exit 1
    fi

    TNS_ADMIN=${TNS_ADMIN:-$MK_CONFDIR}

    test -f ${TNS_ADMIN}/sqlnet.ora || ( echo "ORA-99998 Couldn't find "${TNS_ADMIN}/sqlnet.ora ; exit 1)

    export ORACLE_HOME TNS_ADMIN ORACLE_SID LD_LIBRARY_PATH
}

function get_oraversion () {
    # oraenv is only needed when version is determined from sqlplus
    set_oraenv ${1}

    if [[ ! "$1" =~ ^REMOTE_INSTANCE_.* ]] ; then
        # we get the ORACLE_VERSION in main loop because this is only avalible from mk_oracle.cfg
        # that file is only read in function sqlplus at later time!

        # get the version from ORACLE_HOME/bin/sqlplus
        ORACLE_VERSION=$($ORACLE_HOME/bin/sqlplus -V | grep ^SQL | cut -d" " -f3 | cut -d"." -f-2)
    fi

    # remove possible existing variables
    unset AT_LEAST_ORACLE_180 AT_LEAST_ORACLE_122 AT_LEAST_ORACLE_121 AT_LEAST_ORACLE_112 AT_LEAST_ORACLE_111 AT_LEAST_ORACLE_102 AT_LEAST_ORACLE_101 AT_LEAST_ORACLE_92

    if [ "$ORACLE_VERSION" = '18.0' ] ; then
        AT_LEAST_ORACLE_180=yes
    fi

    if [ "$ORACLE_VERSION" = '18.0' -o "$ORACLE_VERSION" = '12.2' ] ; then
        AT_LEAST_ORACLE_122=yes
    fi

    if [ "$ORACLE_VERSION" = '18.0' -o "$ORACLE_VERSION" = '12.2' -o "$ORACLE_VERSION" = '12.1' ] ; then
        AT_LEAST_ORACLE_121=yes
    fi

    if [ "$ORACLE_VERSION" = '18.0' -o "$ORACLE_VERSION" = '12.2' -o "$ORACLE_VERSION" = '12.1' -o "$ORACLE_VERSION" = '11.2' ] ; then
        AT_LEAST_ORACLE_112=yes
    fi

    if [ "$ORACLE_VERSION" = '18.0' -o "$ORACLE_VERSION" = '12.2' -o "$ORACLE_VERSION" = '12.1' -o "$ORACLE_VERSION" = '11.2' \
         -o "$ORACLE_VERSION" = '11.1' ] ; then
        AT_LEAST_ORACLE_111=yes
    fi

    if [ "$ORACLE_VERSION" = '18.0' -o "$ORACLE_VERSION" = '12.2' -o "$ORACLE_VERSION" = '12.1' -o "$ORACLE_VERSION" = '11.2' \
         -o "$ORACLE_VERSION" = '11.1' -o "$ORACLE_VERSION" = '10.2' ] ; then
        AT_LEAST_ORACLE_102=yes
    fi

    if [ "$ORACLE_VERSION" = '18.0' -o "$ORACLE_VERSION" = '12.2' -o "$ORACLE_VERSION" = '12.1' -o "$ORACLE_VERSION" = '11.2' \
         -o "$ORACLE_VERSION" = '11.1' -o "$ORACLE_VERSION" = '10.2' \
         -o "$ORACLE_VERSION" = '10.1' ] ; then
        AT_LEAST_ORACLE_101=yes
    fi

    if [ "$ORACLE_VERSION" = '18.0' -o "$ORACLE_VERSION" = '12.2' -o "$ORACLE_VERSION" = '12.1' -o "$ORACLE_VERSION" = '11.2' \
         -o "$ORACLE_VERSION" = '11.1' -o "$ORACLE_VERSION" = '10.2' \
         -o "$ORACLE_VERSION" = '10.1' -o "$ORACLE_VERSION" = '9.2' ] ; then
        AT_LEAST_ORACLE_92=yes
    fi
}

#.
#   .--Functions-----------------------------------------------------------.
#   |             _____                 _   _                              |
#   |            |  ___|   _ _ __   ___| |_(_) ___  _ __  ___              |
#   |            | |_ | | | | '_ \ / __| __| |/ _ \| '_ \/ __|             |
#   |            |  _|| |_| | | | | (__| |_| | (_) | | | \__ \             |
#   |            |_|   \__,_|_| |_|\___|\__|_|\___/|_| |_|___/             |
#   |                                                                      |
#   +----------------------------------------------------------------------+
#   | Helper functions                                                     |
#   '----------------------------------------------------------------------'


function ora_session_environment()
{
    echo 'set pages 0 trimspool on feedback off lines 8000'
    if [ "$AT_LEAST_ORACLE_102" = 'yes' -a ! "$DISABLE_ORA_SESSION_SETTINGS" ] ; then
        echo 'set echo off'
        echo 'alter session set "_optimizer_mjc_enabled"=false;'

        # cursor_sharing is not valid for ASM instances
        if [ ! ${ORACLE_SID:0:1} = '+' ] ; then
            echo 'alter session set cursor_sharing=exact;'
        fi

        echo 'set echo on'
    fi
    echo 'whenever sqlerror exit 1'
    echo ' '
}

# Helper function that calls an SQL statement with a clean output
# Usage: echo "..." | sqlplus SID
function sqlplus() {
    loc_stdin=$(cat)
    local SID=$1


    # reload mk_oracle.cfg for run_cached. Otherwise some variables are missing
    if [ -e "$MK_CONFDIR/mk_oracle.cfg" ]
    then
        . $MK_CONFDIR/mk_oracle.cfg
    fi

    ORADBUSER=""
    DBPASSWORD=""

    set_oraenv $SID

    if [[ "$1" =~ ^REMOTE_INSTANCE_.* ]] ; then

        # working on REMOTE_-Mode!
        ORACFGLINE=$(eval echo \${$1})

        ORACLE_SID=$(echo ${ORACFGLINE} | cut -d":" -f$[7])

        TNSALIAS=$(echo $REMOTE_VARNAME | cut -d"_" -f3-)

        # we need to add the piggyback sections!
        remote_hostname=$(echo ${ORACFGLINE} | cut -d":" -f$[6])

        # build the piggyback information in loc_stdin
        # <<<<hostname>>>> SQL-Statements <<<<>>>>
        loc_stdin=$(echo 'prompt <<<<'$remote_hostname'>>>>'; echo ' ')$loc_stdin$(echo ' '; echo 'prompt <<<<>>>>'; echo ' ')
    else
        # working with locally running instances
        # mk_oracle_dbusers.conf is for compatibility. Do not use it anymore
        ORACLE_USERCONF=${MK_CONFDIR}/mk_oracle_dbuser.conf

        TNSALIAS=${ORACLE_SID}

        # ASM use '+' as 1st character in SID!
        if [ ${ORACLE_SID:0:1} = '+' ] ; then
            ORACFGLINE=${ASMUSER}
        else
            # use an individuel user or the default DBUSER from mk_oracle.cfg
            dummy="DBUSER_"${ORACLE_SID}
            ORACFGLINE=${!dummy}
            if [ "$ORACFGLINE" = '' ] ; then
                ORACFGLINE=${DBUSER}
            fi
        fi

        if [ -f ${ORACLE_USERCONF} -a "${ORACFGLINE}" = '' ] ; then
            # mk_oracle_dbuser.conf
            ORACFGLINE=$(cat ${ORACLE_USERCONF} | grep "^"${ORACLE_SID}":")
            # mk_oracle_dbuser has ORACLE_SID as 1. parameter. we need an offset for all values
            offset=1
        else
            # mk_oracle.cfg
            offset=0
        fi
    fi

    # SID_UPPER is required for later use in function sqlplus.
    # gathering at this point is needed due to dependency to mk_oracle.cfg in remote mode
    SID_UPPER=$(echo $ORACLE_SID | tr '[:lower:]' '[:upper:]')
    export SID_UPPER

    offset=${offset:-0}
    ORADBUSER=$(echo ${ORACFGLINE} | cut -d":" -f$[1+offset])
    DBPASSWORD=$(echo ${ORACFGLINE} | cut -d":" -f$[2+offset])
    DBSYSCONNECT=$(echo ${ORACFGLINE} | cut -d":" -f$[3+offset])
    DBHOST=$(echo ${ORACFGLINE} | cut -d":" -f$[4+offset])
    DBPORT=$(echo ${ORACFGLINE} | cut -d":" -f$[5+offset])

    TNSPINGOK=no
    if [ -f ${TNS_ADMIN}/tnsnames.ora ] ; then
        if "${ORACLE_HOME}"/bin/tnsping "${TNSALIAS}" >/dev/null 2>&1  ; then
            TNSPINGOK=yes
        else
            unset TNSALIAS
        fi
    else
        unset TNSALIAS
    fi

    if [ ! "${ORACFGLINE}" ] ; then
        # no configuration found
        # => use the wallet with tnsnames.ora or EZCONNECT
        TNSALIAS=${TNSALIAS:-"localhost:1521/${ORACLE_SID}"}
    else
        if [ ${DBSYSCONNECT} ] ; then
            assysdbaconnect=" as "${DBSYSCONNECT}
        fi

        TNSALIAS=${TNSALIAS:-"(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=${DBHOST:-"localhost"})(PORT=${DBPORT:-1521}))(CONNECT_DATA=(SID=${ORACLE_SID})(SERVER=DEDICATED)(UR=A)))"}

        # ORADBUSER = '/'? => ignore DBPASSWORD and use the wallet
        if [ "${ORADBUSER}" = '/' ] ; then
            # connect with / and wallet
            ORADBUSER=""
            DBPASSWORD=""
            if [ "$TNSPINGOK" = 'no' ] ; then
                # create an EZCONNECT string when no tnsnames.ora is usable
                # defaults to localhost:1521/<ORACLE_SID>
                TNSALIAS="${DBHOST:-"localhost"}:${DBPORT:-1521}/${ORACLE_SID}"
            fi
        fi
    fi

    DBCONNECT="${ORADBUSER}/${DBPASSWORD}@${TNSALIAS}${assysdbaconnect}"

    SQLPLUS=${ORACLE_HOME}/bin/sqlplus
    if [ ! -x ${SQLPLUS} ] ; then
        echo "sqlplus not found or ORACLE_HOME wrong! "
        echo "SQLPLUS="${SQLPLUS}
        return 1
    fi

    # add ora_session_environment as prefix before loc_stdin
    loc_stdin=$(ora_session_environment)${loc_stdin}

    if OUTPUT=$(echo "$loc_stdin"  | "$SQLPLUS" -L -s ${DBCONNECT})
    then
        echo "$OUTPUT"
    else

        if [ "$DEBUGCONNECT" ] ; then
            echo "Logindetails: ${DBCONNECT}" >&2
            echo "$OUTPUT"
        else
            # SID_UPPER is required for backword compatibilty to old sqlplus.sh.
            # THis will be removed in a future time
            SID_UPPER=${SID_UPPER:-$(echo $SID | tr '[:lower:]' '[:upper:]')}

            # we need to add the piggyback as the sql didn't return any piggyback information in this situation
            if [[ "$SID" =~ ^REMOTE_INSTANCE_.* ]] ; then
                echo '<<<<'$remote_hostname'>>>>'
            fi

            echo '<<<oracle_instance:sep(124)>>>'
            echo "$OUTPUT" | grep -v "^ERROR at line" | tr '\n' ' ' | sed "s/^/$SID_UPPER|FAILURE|/" ; echo

            if [[ "$SID" =~ ^REMOTE_INSTANCE_.* ]] ; then
                echo '<<<<>>>>'
            fi
        fi
    fi
}

function remove_excluded_sections ()
{
    local sections="$1"
    local excluded="$2"
    local result=""
    for section in $sections
    do
        local skip=
        for exclude in $excluded
        do
            if [ "$exclude" = "$section" ] ; then
                local skip=yes
                break
            fi
        done
        if [ "$skip" != yes ] ; then
            result="$result $section"
        fi
    done
    echo "$result"
}


# Create one SQL statements for several sections and run
# these with sqlplus. The exitcode is preserved.
function do_sync_checks ()
{
    local SID=$1
    local SECTIONS="$2"
    for section in $SECTIONS
    do
        eval "sql_$section"
    done | sqlplus $SID
}

function do_async_checks ()
{
    local SID=$1
    echo "$ASYNC_SQL" | sqlplus $SID
}

# Make sure that the new shell that is being run by run_cached inherits
# our functions
export -f sqlplus
export -f ora_session_environment
export -f do_async_checks
export -f set_oraenv

function file_mtime() {
    /usr/bin/perl -e 'if (! -f $ARGV[0]){die "0000000"};$mtime=(stat($ARGV[0]))[9];print ($mtime);' "$1"
}

function run_cached () {
    local section=
    if [ "$1" = -s ] ; then local section="echo '<<<$2>>>' ; " ; shift ; fi
    local NAME=$1
    local MAXAGE=$2
    shift 2
    local CMDLINE="$section$@"

    if [ ! -d $MK_VARDIR/cache ]; then mkdir -p $MK_VARDIR/cache ; fi
    CACHEFILE="$MK_VARDIR/cache/$NAME.cache"

    # Check if the creation of the cache takes suspiciously long and return
    # nothing if the age (access time) of $CACHEFILE.new is twice the MAXAGE
    # perl is needed for Solaris => no date +%s availible
    local NOW=$(perl -le "print time()")
    if [ -e "$CACHEFILE.new" ] ; then
        local CF_ATIME=$($STATCX "$CACHEFILE.new")
        if [ $((NOW - CF_ATIME)) -ge $((MAXAGE * 2)) ] ; then
            # Kill the process still accessing that file in case
            # it is still running. This avoids overlapping processes!
            fuser -k -9 "$CACHEFILE.new" >/dev/null 2>&1
            rm -f "$CACHEFILE.new"
            return
        fi
    fi

    # Check if cache file exists and is recent enough
    if [ -s "$CACHEFILE" ] ; then
        local MTIME=$($STATCY "$CACHEFILE")
        if [ $((NOW - MTIME)) -le $MAXAGE ] ; then local USE_CACHEFILE=1 ; fi
        # Output the file in any case, even if it is
        # outdated. The new file will not yet be available
        cat "$CACHEFILE"
    fi

    # Cache file outdated and new job not yet running? Start it
    if [ -z "$USE_CACHEFILE" -a ! -e "$CACHEFILE.new" ] ; then
        if [ "$DEBUG" ] ; then
            echo "set -o noclobber ; exec > \"$CACHEFILE.new\" || exit 1 ; $CMDLINE && mv \"$CACHEFILE.new\" \"$CACHEFILE\" || rm -f \"$CACHEFILE\" \"$CACHEFILE.new\"" | /bin/bash
        else
            # When the command fails, the output is throws away ignored
            echo "set -o noclobber ; exec > \"$CACHEFILE.new\" || exit 1 ; $CMDLINE && mv \"$CACHEFILE.new\" \"$CACHEFILE\" || rm -f \"$CACHEFILE\" \"$CACHEFILE.new\"" | nohup /bin/bash >/dev/null 2>&1 &
        fi
    fi
}

function do_testmode() {
    echo "-----------------------------------------------"
    echo "Operating System: "${ostype}
    echo "Logincheck to Instance: "$SID"  Version: "$ORACLE_VERSION
    echo "select 'Login ok User: ' || user || ' on ' || host_name ||' Instance ' || instance_name
          from v\$instance;" | sqlplus $SID
    echo "SYNC_SECTIONS=$SECTIONS"
    echo "ASYNC_SECTIONS=$ASECTIONS"
    if [ "$IGNORE_DB_NAME" ] ; then
        echo "IGNORE_DB_NAME found. Ignoring DB_NAME in all SQLs!"
    fi
    if [ "$DISABLE_ORA_SESSION_SETTINGS" ] ; then
        echo "Paramter DISABLE_ORA_SESSION_SETTINGS found!"
    fi
    if [ "$HINT_RMAN" ] ; then
        echo "Using HINT_RMAN for this Instance!"
    fi
}

function do_instance() {

    SID=$1

    if [ ${ORACLE_SID:0:1} = '+' ] ; then
        DO_ASYNC_SECTIONS=${ASYNC_ASM_SECTIONS}
        DO_SYNC_SECTIONS=${SYNC_ASM_SECTIONS}
    else
        # switch sections to ASM
        DO_SYNC_SECTIONS=${SYNC_SECTIONS}
        DO_ASYNC_SECTIONS=${ASYNC_SECTIONS}
    fi

    # Do sync checks
    EXCLUDED=$(eval 'echo $EXCLUDE'"_$SID")
    SECTIONS=$(remove_excluded_sections "$DO_SYNC_SECTIONS" "$EXCLUDED")

    # Do async checks
    ASECTIONS=$(remove_excluded_sections "$DO_ASYNC_SECTIONS" "$EXCLUDED")
    ASYNC_SQL=$(for section in $ASECTIONS ; do eval "sql_$section" ; done)
    export ASYNC_SQL

    if [ "$DEBUGCONNECT" ] ; then
        do_testmode $SID
    else
        do_sync_checks $SID "$SECTIONS"
        run_cached oracle_$SID $CACHE_MAXAGE do_async_checks $SID
    fi
}

#.
#   .--Main----------------------------------------------------------------.
#   |                        __  __       _                                |
#   |                       |  \/  | __ _(_)_ __                           |
#   |                       | |\/| |/ _` | | '_ \                          |
#   |                       | |  | | (_| | | | | |                         |
#   |                       |_|  |_|\__,_|_|_| |_|                         |
#   |                                                                      |
#   +----------------------------------------------------------------------+
#   |  Iterate over all instances and execute sync and async sections.     |
#   '----------------------------------------------------------------------'

# set some basic operating system stuff
set_osenv

# Are there any remote configurations?
for element in $(compgen -A variable | ${GREP} -E "^REMOTE_INSTANCE_.*") ; do

    REMOTE_DBS=$REMOTE_DBS" "$element

    remote_hostname=$(echo $element | cut -d":" -f6)

    REMOTE_HOSTLIST=${remote_hostname}" "${REMOTE_HOSTLIST}
done

if [ "$REMOTE_HOSTLIST" ] ; then
    # remove duplicate hosts from list
    REMOTE_HOSTLIST=$(echo $REMOTE_HOSTLIST | tr ' ' '\n' | sort | uniq)

    # create empty piggyback SECTIONS
    for element in $REMOTE_HOSTLIST ; do

        remote_hostname=$(echo $element | cut -d":" -f6)
        echo "<<<<"$remote_hostname">>>>"

        for section in $SYNC_SECTIONS $ASYNC_SECTIONS $SYNC_ASM_SECTIONS $ASYNC_ASM_SECTIONS
        do
            echo "<<<oracle_"$section">>>"
        done

        echo "<<<<>>>>"
    done

    if [ ! -e "$MK_VARDIR/mk_oracle.found" ] ; then
        touch "$MK_VARDIR/mk_oracle.found"
    fi
fi

# Get list of all running databases
# Do not work on ASM in this plugin. => Ignore a running ASM-Instance!
SIDS=$(UNIX95=true ps -ef | ${AWK} '{print $NF}' | ${GREP} -E '^asm_pmon_|^ora_pmon_|^xe_pmon_XE' | cut -d"_" -f3-)

# If we do not have found any running database instance, then either
# no ORACLE is present on this system or it's just currently not running.
# In the later case we ouput empty agent sections so that Check_MK will be
# happy and execute the actual check functions.
if [ -z "$SIDS" -a ! -e "$MK_VARDIR/mk_oracle.found" ] ; then
    exit
fi

# From now on we expect databases on this system (for ever)
touch $MK_VARDIR/mk_oracle.found

# Make sure that always all sections are present, even
# in case of an error. Note: the section <<<oracle_instance>>>
# section shows the general state of a database instance. If
# that section fails for an instance then all other sections
# do not contain valid data anyway.
for section in $SYNC_SECTIONS $ASYNC_SECTIONS $SYNC_ASM_SECTIONS $ASYNC_ASM_SECTIONS
do
    echo "<<<oracle_"$section">>>"
done

for SID in $SIDS
do

    # Check if SID is listed in ONLY_SIDS if this is used
    if [ "$ONLY_SIDS" ] ; then
        SKIP=yes
        for S in $ONLY_SIDS ; do
            if  [ "$S" = "$SID" ] ; then
                SKIP=
                break
            fi
        done
        if [ "$SKIP" ] ; then continue ; fi
    fi

    EXCLUDE=EXCLUDE_$SID
    if [[ "$EXCLUDE" =~ ^[a-zA-Z][a-zA-Z0-9_]*$ ]]
    then
        # Handle explicit exclusion of instances
        # but not for +ASM
        EXCLUDE=${!EXCLUDE}
        # SID filtered totally?
        if [ "$EXCLUDE" = "ALL" ]; then
            continue
        fi
    fi

    get_oraversion $SID
    do_instance $SID
done

for REMOTE_DB in $REMOTE_DBS ; do
    REMOTE_DB_LINE=$(eval echo \${$REMOTE_DB})

    # ORACLE_VERSION comes from mk_oracle.cfg!
    ORACLE_VERSION=$(echo $REMOTE_DB_LINE | cut -d":" -f8)

    # the ORACLE_SID is needed for the oracle_instance check for ASM and normal instance
    ORACLE_SID=$(echo $REMOTE_DB_LINE | cut -d":" -f7)

    # This is the piggyback hostname
    remote_hostname=$(echo $element | cut -d":" -f6)

    get_oraversion $REMOTE_DB
    do_instance $REMOTE_DB
done