exhwrf_launch.py

#! /usr/bin/env python

##@namespace scripts.exhwrf_launch
# Creates the initial HWRF directory structure for executing a
# single HWRF cycle.  This script must be run before any other.
#
# This script is executed as follows:
# @code{.sh}
# exhwrf_launch.py YYYYMMDDHH STID CASE_ROOT /path/to/parm [options]
# @endcode
#
# @note When NCEP Central Operations (NCO) runs this job, the $PARAFLAG
#   environment variable must be set to "NO" to trigger NCO-specific rules
#
# Command line argument meanings:
#
# * YYYYMMDDHH --- a ten digit cycle date and hour
# * STID --- a three character storm identifier: storm number and one
#   letter basin.  For example, 12L for Katrina.
# * CASE_ROOT --- HISTORY for a retrospective run and FORECAST for
#   a real-time run
# * /path/to/parm --- path to the parm/ directory for locating the
#   standard *.conf files
#
# The [options] can be:
#
# * file.conf --- a configuration file to read
# * section.Option=VALUE --- after reading all configuration files,
#   set option "Option" in section "section" to the given value.
#
# This is the order in which configuration files and options are
# processed:
#
# 1. Internal initial options from the startdata variable (which does almost nothing).
# 2. parm/hwrf_input.conf --- input file locations
# 3. parm/hwrf.conf --- detailed HWRF configuration settings
# 4. parm/hwrf_holdvars.conf --- for generating the com/storm*.holdvars.txt file
# 5. parm/hwrf_basic.conf --- basic, high-level HWRF configuration settings
# 6. Configuration files listed in the [options] to exhwrf_launch
# 7. Configuration section.Option=VALUE settings in exhwrf_launch
# 8. The hwrf_expt.prelaunch() function is called to set per-cycle or per-basin settings.
#
# See the hwrf_expt.prelaunch() function and the hwrf.prelaunch module
# for details on the prelaunch functionality.
#
# After configuration information is determined, the sanity checks are
# run.  If the sanity checks succeed, the initial directory structure is
# created and the com/storm*.conf file is generated.  The database is
# filled with products and tasks generated by the hwrf_expt module, and
# then the script exits.

import os, sys, re, logging, collections

if 'USHhwrf' in os.environ:
    sys.path.append(os.environ['USHhwrf'])
elif 'HOMEhwrf' in os.environ:
    sys.path.append(os.path.join(os.environ['HOMEhwrf'],'ush'))
else:
    guess_HOMEhwrf=os.path.dirname(os.path.dirname(
            os.path.realpath(__file__)))
    guess_USHhwrf=os.path.join(guess_HOMEhwrf,'ush')
    sys.path.append(guess_USHhwrf)

import produtil.setup, produtil.log, produtil.dbnalert
import hwrf.launcher
import hwrf_expt
from hwrf.numerics import to_datetime

## The logging.Logger for log messages
logger=None

## Initial configuration data to be inserted to the
# hwrf.launcher.HWRFLauncher before reading configuration files.
startdata='''
# Holdvars file with ksh variables:
holdvars="{holdvars}"

# Main conf file:
CONFhwrf="{CONFhwrf}"

# Cycle being run:
cycle={YMDH}

# Three character storm ID -- just number and basin letter:
stormid3="{vit[stormid3]}"

# Long storm ID:
longstormid="{vit[longstormid]}"
'''  # Don't forget the end of line before the '''

def usage(logger):
    logger.critical('Invalid arguments to exhwrf_launch.py.  Aborting.')
    print '''
Usage: exhwrf_launch.py 2014062400 95E case_root /path/to/parm [options]

Mandatory arguments: 
  2014062400 -- the cycle to run
  95E -- storm id
  case_root -- FORECAST = real-time mode, HISTORY = retrospective mod
  /path/to/parm -- location of parm directory where standard conf files
      reside

Optional arguments:
section.option=value -- override conf options on the command line
/path/to/file.conf -- additional conf files to parse

Aborting due to incorrect arguments.'''
    sys.exit(2)

def main():
    """!Processes configuration information and passes on to the
    hwrf.launcher module to create the initial directory structure and
    conf file."""
    logger=logging.getLogger('exhwrf_launch')
    PARAFLAG = ( 'YES' == os.environ.get('PARAFLAG','YES') )
    logger.info('Top of exhwrf_launch.')

    args=sys.argv[1:]

    # Check the initial arguments passed in.
    if len(args)<4: usage(logger)

    # Find cycle: Same for  all storms in a multistorm.
    cycle=to_datetime(args[0])

    # Multistorm - jtf
    fake_stid = None
    go_since_multistorm_sids = False

    # Test to see if BASINS and/or MULTISTORM_SIDS is set
    basins          = os.environ.get('BASINS', None)
    multi_sids_env  = os.environ.get('MULTISTORM_SIDS','')
    multi_sids = list()
    if multi_sids_env:
        multi_sids = [s for s in multi_sids_env.split()]
    if basins is not None:
        go_since_multistorm_sids = True
        logger.info('Looks like this is rocoto, running a multistorm with basins: %s'%(basins))
        # call storm priority
        bstorms = hwrf.launcher.multistorm_priority(args, basins, logger, usage)
        logger.info('Priority found the following storms: ' +repr(bstorms))
        for s in bstorms:
            if s not in multi_sids:
                multi_sids.append(s)

    logger.info('MS LIST: ' +repr(multi_sids))

    if multi_sids:
        go_since_multistorm_sids = True
    fakestorm_conf = None

    if go_since_multistorm_sids:    
        logger.info('Parsing input arguments for a multistorm with ids: %s.'% (multi_sids))
        (case_root, parm, infiles, stids, fake_stid, priority_stid, moreopts) = \
                hwrf.launcher.multistorm_parse_args(multi_sids, args[1:], logger, usage)
        # Make sure you pass the last elements of the moreopts list, since it is the options
        # for the fake storm, namely the correct config.startfile.
        fakestorm_conf=hwrf.launcher.launch(infiles,cycle,fake_stid,moreopts[-1],case_root,
                                  prelaunch=hwrf_expt.prelaunch,
                                  fakestorm=True)
    else:
        (case_root,parm,infiles,stid,moreopt) = \
            hwrf.launcher.parse_launch_args(args[1:],logger,usage)
        stids = [stid]
        moreopts = [moreopt]

        logger.info('Requested storm %s cycle %s case root %s'
                    %(stid,cycle.strftime('%Y%m%d%H'),case_root))       

    global_storm_num = 2
    for i,stid in enumerate(stids):
        if stid != fake_stid:
            conf=hwrf.launcher.launch(infiles,cycle,stid,moreopts[i],case_root,
                                      prelaunch=hwrf_expt.prelaunch,
                                      fakestorm_conf=fakestorm_conf,
                                      storm_num=global_storm_num)
        else:
            conf=fakestorm_conf

        global_storm_num += 1
        conf.sanity_check()

        if 'NO'==os.environ.get('PARAFLAG','YES'):
            message=conf.strinterp('wcoss_fcst_nco','{messages}/message{storm_num}')
            if os.path.exists(message):
                alert=produtil.dbnalert.DBNAlert(['MODEL','HWRF_MESSAGE','{job}',
                                                  message])
                alert()

        holdvars=conf.strinterp('dir','{com}/{stormlabel}.holdvars.txt')
        logger.info(holdvars+': write holdvars here')
        with open(holdvars,'wt') as f:
            f.write(conf.make_holdvars())

        if conf.has_option('config','startfile'):
            startfile=conf.getstr('config','startfile')
            logger.info(startfile+': Write holdvars and conf location here.')
            startcontents=conf.strinterp('config',startdata,holdvars=holdvars)
            with open(startfile,'wt') as f:
                f.write(startcontents)


if __name__ == '__main__': 
    try:
        produtil.setup.setup()
        produtil.log.postmsg('exhwrf_launch is starting')
        main()
        produtil.log.postmsg('exhwrf_launch completed')
    except Exception as e:
        produtil.log.jlogger.critical(
            'exhwrf_launch failed: %s'%(str(e),),exc_info=True)
        sys.exit(2)