exhwrf_input.py

#! /usr/bin/env python

##@namespace scripts.exhwrf_input
# Pulls input data from tape or over the network.
#
# This script will examine the selected HWRF configuration to figure out
# what data is needed to run it.  This is done by calling
# hwrf_expt.inputiter().  The result is sent to an
# hwrf.input.InputSource for processing.  The InputSource object will
# pull data from multiple sources in priority order, as defined in the
# hwrf configuration files (typically parm/hwrf_input.conf)

import logging, os, sys
import produtil.log, produtil.setup, produtil.run, produtil.fileop
from produtil.log import jlogger
from produtil.run import batchexe
import hwrf.input

def main(args):
    """!Uses an hwrf.input.InputSource to get data requested by the
    HWRF configuration (hwrf_expt.inputiter()).  Will exit with status
    1 if any mandatory data is missing."""
    import hwrf_expt
    hwrf_expt.init_module(make_ensemble_da=True)

    conf=hwrf_expt.conf
    cycle=hwrf_expt.conf.cycle
    input_catalog=conf.get('config','input_catalog')
    input_sources=conf.get('config','input_sources')
    logger=conf.log('exhwrf_input')
    WORKhwrf=conf.getdir('WORKhwrf')

    if input_catalog!='hwrfdata':
        jlogger.info("Input catalog is %s, not \"hwrfdata\" so data should "
                     "be staged on disk already.  I have nothing to do, so "
                     "I'll just exit.  This is not an error.")
        sys.exit(0)

    # Make sure we're in the cycle's work directory, otherwise we might
    # pull archives and other big things to $HOME.
    produtil.fileop.chdir(WORKhwrf,logger=logger)

    # Figure out how to run htar:
    htar=batchexe(conf.getexe('htar'))

    # Figure out how to run hsi:
    hsi=batchexe(conf.getexe('hsi'))

    # Get the list of data to pull:
    indata=list(d for d in hwrf_expt.inputiter())

    nostage=True
    skipme=set()
    onlyme=set()
    for arg in args:
        if arg[0:5]=='skip:':
            logger.info('%s: skip dataset %s'%(arg,arg[5:]))
            skipme.add(arg[5:])
        elif arg[0:5]=='only:':
            logger.info('%s: only pull dataset %s'%(arg,arg[5:]))
            onlyme.add(arg[5:])
        elif arg == '--stage':
            logger.info('--stage: enable -Hnostage')
            nostage=False
        elif arg=='-Hnostage' or arg=='--nostage' or arg=='--no-stage':
            logger.info('%s: do not use -Hnostage'%(arg,))
            nostage=True
        else:
            logger.warning('Ignoring unrecognized argument %s'%(repr(arg),))

    # Make the final list:
    data=list()
    for d in indata:
        ds=d['dataset']
        if ds in skipme or (onlyme and (ds not in onlyme)):
            pass #logger.info('SKIP: %s'%(repr(d),))
        else:
            #logger.info('FIND: %s'%(repr(d),))
            data.append(d)

    # Decide where to put the data:
    cat=hwrf.input.DataCatalog(conf,"hwrfdata",cycle)

    if nostage: 
        jlogger.info('Enabling htar -Hnostage option.')
        htar=htar['-Hnostage']
    else:
        jlogger.info('Not using htar -Hnostage option.')

    # Now pull the data:
    getem=hwrf.input.InputSource(conf,input_sources,conf.cycle,
                                 htar=htar,hsi=hsi,logger=logger)
    bad=not getem.get(data,cat)
    if bad:
        jlogger.error('Missing data in exhwrf_input.  Workflow may fail.')
        sys.exit(1)

if __name__=='__main__':
    try:
        produtil.setup.setup(thread_logger=True,eloglevel=logging.INFO)
        jlogger.info("HWRF input job starting")
        main(sys.argv[1:])
        jlogger.info("HWRF input job completed")
    except Exception as e:
        jlogger.critical('HWRF input is aborting: '+str(e),exc_info=True)
        sys.exit(2)