from   collections import OrderedDict
from   datetime    import datetime, timezone
import sys

try:
    import yaml
    assert yaml.__version__ > '5.1'
except:
    sys.exit(
        'package "PyYAML" (>= 5.1) is required but not installed! Run\n' +
        'python -m pip install --user pyyaml\n' +
        'to install it.'
        )

import palm
from   palm.check      import *
from   palm.PalmConfig import PalmConfig
import palm.system


LOG = palm.logger(__name__)


def yaml_process( yaml, options, rules=None ):

    if yaml is None:
        yaml = {}

    # read and expand defintions ...
    #
    yamldefs, palmdefs = yaml_expand(
        yaml,
        options,
        rules=rules,
        override=False,
        validate=False,
        include_defaults=True
        )

    palmdefs.override = True
    palmdefs.validate = False # TODO: need to be discussed

    # NOTE: there is always a palm configuration file
    #       while we use old and new parts of the
    #       software system
    #
    palmdefs.config_file = palm.system.palm_config_file(
        palmdefs.configuration_identifier
        )

    return yamldefs, palmdefs


def yaml_to_palm( yaml, palmstream, context=None, rules=None ):

    if context is None:
        context = {}

    palmdefs = yaml
    if not isinstance(palmdefs, PalmConfig ):
        ignore, palmdefs = yaml_process( yaml, context, rules )

    def write_palm( stream ):

        now = datetime.now( timezone.utc ).strftime("%Y/%m/%d %H:%M:%S UTC")
        stream.write( '# autogenerated by palm-cli: {}\n'.format(now) )
        stream.write( '#\n'                                           )

        multiliner = []

        for rule in palmdefs.rules:
            name = rule.name
            if name is None:
                continue

            value = palmdefs.get(name)
            if value is not None:
                value = rule.format( name, value ).strip()
                if not value:
                    pass
                elif rule.format == FORMAT_PREFIX:
                    multiliner.append(value)
                else:
                    stream.write( value )
                    stream.write( '\n'  )

        for value in multiliner:
            stream.write( '\n'  )
            stream.write( value )
            stream.write( '\n'  )


    # converting ...
    #
    try:
        LOG.debug( 'start converting yaml to palm ...' )

        # write palm config file
        #
        if isinstance( palmstream, str ):
            LOG.debug( 'start writing: %s ...', palmstream )
            with open( palmstream, 'w', encoding = 'utf-8' ) as palmstream:
                write_palm( palmstream )

        else:
            write_palm( palmstream )

        LOG.debug( 'stop  converting yaml to palm ...' )

    except:
        LOG.error( 'fail  converting yaml to palm ...' )
        raise


def yaml_test( yaml, *context, rules=None ):

    yamldefs = yaml_read( yaml )
    if 'logging' in yamldefs:
        palm.init_logging( yamldefs.get('logging') )
        del yamldefs['logging']

    result = []
    yamldefs, palmdefs = yaml_expand(
        yamldefs,
        *context,
        rules=rules,
        override=False,
        validate=True,
        report=result
        )

    return result


def yaml_expand( yaml, *context, rules=None, override=False, validate=False, report=None, include_defaults=True ):

    if not isinstance( yaml, dict ):
        yaml = yaml_read( yaml )

    if rules is None:
        rules = __PALM_RULES__

    config = PalmConfig(
        *context,
        rules=rules,
        override=False,
        validate=validate if report is None else True
        )

    def expand_and_add( name, value ):
        try:
            aliases = config.aliases(name)

            LOG.trace( '%s:',  ', '.join(aliases) )
            LOG.trace( '> %s', value if not isinstance(value,str) else "'{}'".format(value) )

            if type(value) is str:
                value = config.expand( value, ignore=aliases )
                LOG.trace( '= %s', value if not isinstance(value,str) else "'{}'".format(value) )

            config[name] = value # and validate if set

        except AssertionError as error:
            if report is None:
                raise
            else:
                report.append(error)

    try:
        LOG.debug( 'start expanding yaml definitions ...')

        for path, value, group in yaml_iterate(yaml):

            yamlname = yaml_name(path)
            if config.managing(yamlname):
                continue

            expand_and_add( yamlname, value )

        LOG.debug( 'stop  expanding yaml definitions ...')

    except Exception as exception:
        LOG.error( 'fail  expanding yaml definitions ...')
        raise

    if include_defaults:
        try:
            LOG.debug( 'start expanding default definitions ...')

            for rule in config.rules:

                name = rule.pathname
                if name is None:
                    name = rule.name

                if config.managing(name):
                    continue

                expand_and_add( name, rule.default )

            LOG.debug( 'stop  expanding default definitions ...')

        except Exception as exception:
            LOG.error( 'fail  default yaml definitions ...')
            raise

    return yaml, config


def yaml_iterate( yaml ):

    if not isinstance( yaml, dict ):
        yaml = yaml_read( yaml )

    # internal (recursive) deep-first-search generator ...
    #
    def dfs( yaml, path=None ):
        path = [] if path is None else path.copy()
        for name, value in yaml.items():

            path.append( name )
            if isinstance( value, dict ):
                yield from dfs( value, path )
            else:
                yield path, value, yaml
            path.pop()

    return dfs( yaml )


def yaml_read( stream ):

    # internal loading func. for ordered loading ...
    #
    def yaml_load_ordered( stream, Loader=yaml.Loader, object_pairs_hook=OrderedDict ):
        class OrderedLoader(Loader):
            pass

        def construct_mapping(loader, node):
            loader.flatten_mapping(node)
            return object_pairs_hook(loader.construct_pairs(node))

        OrderedLoader.add_constructor(
            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
            construct_mapping
            )

        return yaml.load(stream, OrderedLoader)

    # REQUIRE: stream is String | Stream
    #
    try:
        LOG.debug( 'start reading yaml definitions ...')

        result = {}
        if isinstance( stream, str ):
            LOG.debug( 'start reading: %s ...', stream )
            with open( stream ) as stream:
                result = yaml_load_ordered( stream, Loader=yaml.FullLoader )

        else:
            result = yaml_load_ordered( stream, Loader=yaml.FullLoader )

        LOG.debug( 'stop  reading yaml definitions ...')

        return result

    except Exception as exception:
        LOG.error( 'fail  reading yaml definitions ...')
        raise


def yaml_get( yaml, *yamlpath, default=None ):

    # define useful recursive dict.get ...
    #
    def get( yaml, yamlpath ):
        result = yaml
        for name in yamlpath:
            if result is default or result == default:
                break

            # TODO:
            result = result.get( name, default )

        return result

    # REQUIRE: yaml is dict
    # REQUIRE: yamlpath is tuple|list|string?

    # get value from path ...
    #
    result = get( yaml, __p2t__(*yamlpath) )
    return result


def yaml_name( *yamlpath, delim='_' ):
    result = delim.join(__p2t__(*yamlpath))
    return result


def __p2t__( *yamlpath ):
    # todo: yamlpath[0] is string
    if len(yamlpath) == 1 and type(yamlpath[0]) is list:
        yamlpath = tuple(yamlpath[0])

    return yamlpath


def check_arg( arg, name, typeinfo=str ):
    def find_and_check( string ):
        result = None if string is None else typeinfo(string)
        for rule in __PALM_RULES__:
            if rule.name == name or rule.pathname == name:
                try:
                    rule.validate( result, arg, {} )
                except Exception as exception:
                    raise AssertionError( '\n  ++ error: '+ exception.args[0] )
                break
        return result
    #print(arg, name, typeinfo)
    return find_and_check


# result:
#
FORMAT_NONE = lambda name, value: ''


# result: %<name> <value>
#
FORMAT_PAIR = lambda name, value: "%{0:30} {1}".format( name, value )

# result:
#         <name> <value[0]>
#         <name> <value[1]>
#         ...
FORMAT_PREFIX = lambda name, value: '\n' + '\n'.join( [ name + line for line in value.strip().splitlines()])

# result: <name> <value-1>; <value-2>; ...
#
FORMAT_INFIX  = lambda name, value: FORMAT_PAIR( name, '; '.join( value.strip().splitlines()) )


# NOTE: an unset parameter is None, has a <default-value> or is  ...
#
REQUIRED = PalmConfig.Rule.REQUIRED


__PALM_RULES__ = [

    # root
    PalmConfig.Rule(
        'path_root',
        default=palm.system.root_dir()
        ),

    # base ...
    #
    PalmConfig.Rule(
        'base_directory', ['path', 'base'],
        FORMAT_PAIR,
        CHECK_IS_DIRECTORY,
        # FIXME: It is recommanded to use <palm-prefix> as workspace
        #        and as current working directory
        '$PWD'
        ),
    PalmConfig.Rule(
        'base_data', ['path', 'runs'],
        FORMAT_PAIR,
        CHECK_IS_DIRECTORY,
        '${path_base}/JOBS'
        ),
    PalmConfig.Rule(
        'fast_io_catalog', ['path', 'tmp'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        '${path_base}/tmp'
        ),
    PalmConfig.Rule(
        None, ['path','ioFileConnection'],
        FORMAT_PAIR,
        CHECK_IS_FILE,
        # FIXME: location changed
        '${path_root}/share/config'
        ),

    # runfiles ...
    #
    PalmConfig.Rule(
        'runfiles_input', ['runfiles', 'input'],
        FORMAT_PAIR,
        CHECK_IS_DIRECTORY,
        '${path_runs}/${run_identifier}/INPUT'
        ),
    PalmConfig.Rule(
        'runfiles_output', ['runfiles', 'output'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        '${path_runs}/${run_identifier}/OUTPUT'
        ),
    PalmConfig.Rule(
        'runfiles_monitoring', ['runfiles', 'monitoring'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        '${path_runs}/${run_identifier}/MONITORING'
        ),
    PalmConfig.Rule(
        'user_source_path', ['runfiles', 'usercode'],
        FORMAT_PAIR,
        CHECK_IS_DIRECTORY,
        '${path_runs}/${run_identifier}/USER_CODE'
        ),
    PalmConfig.Rule(
        'runfiles_restart', ['runfiles', 'restart'],
        FORMAT_PAIR,
        CHECK_IS_DIRECTORY,
        '${path_tmp}/${run_identifier}/RESTART'
        ),
    PalmConfig.Rule(
        'runfiles_svf', ['runfiles', 'svf'],
        FORMAT_PAIR,
        CHECK_IS_DIRECTORY,
        '${path_tmp}/${run_identifier}/SVF'
        ),

    # ioFileConnection ...
    #
#     PalmConfig.Rule(
#         None, ['ioFileConnection'],
#         FORMAT_PAIR,
#         CHECK_IS_FILE,
#         # FIXME: location changed
#         '${path_root}/share/config/.palm.iofiles'
#         ),

    PalmConfig.Rule(
        'source_path', ['source', 'path'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        '${path_root}/src'
        ),

    # host (local) ...
    #
#TODO delete
#     PalmConfig.Rule(
#         'local_ip', ['host', 'local', 'ip'],
#         FORMAT_PAIR,
#         CHECK_ONE(
#             CHECK_WITH( 'mode', CHECK_MATCH('local') ), # skip for local build
#             CHECK_AND(
#                 CHECK_IS(str),
#                 CHECK_NOT_EMPTY
#                 )
#             ),
#         None # TODO: REQUIRED for remote system
#         ),
#TODO delete
#     PalmConfig.Rule(
#         'local_hostname', ['host', 'local', 'hostname'],
#         FORMAT_PAIR,
#         CHECK_AND(
#             CHECK_IS(str),
#             CHECK_NOT_EMPTY
#             ),
#         '$HOSTNAME'
#         ),
    PalmConfig.Rule(
        'local_username', ['host', 'local', 'username'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        '$USER'
        ),
    PalmConfig.Rule(
        'local_jobcatalog', ['host', 'local', 'logs'],
        FORMAT_PAIR,
        CHECK_ONE(
            CHECK_WITH( 'mode', CHECK_MATCH('remote') ),
            CHECK_AND(
                CHECK_IS(str),
                CHECK_NOT_EMPTY
                )
            ),
        '${path_runs}/${run_identifier}/LOG_FILES'
        ),

    # host (remote) ...
    #
    PalmConfig.Rule(
        'remote_ip', ['host', 'remote', 'ip'],
        FORMAT_PAIR,
        CHECK_ONE(
            CHECK_WITH( 'mode', CHECK_MATCH('local') ),
            CHECK_AND(
                CHECK_IS(str),
                CHECK_NOT_EMPTY
                )
            ),
        None # TODO: REQUIRED on remote system
        ),
    PalmConfig.Rule(
        'loginnode', ['host', 'remote', 'loginnode'],
        FORMAT_PAIR,
        CHECK_ONE(
            CHECK_WITH( 'mode', CHECK_MATCH('local') ),
            CHECK_AND(
                CHECK_IS(str),
                CHECK_NOT_EMPTY
                )
            ),
        None
        ),
#TODO delete
#     PalmConfig.Rule(
#         'remote_hostname', ['host', 'remote', 'hostname'],
#         FORMAT_PAIR,
#         CHECK_ONE(
#             CHECK_WITH( 'mode', CHECK_MATCH('local') ),
#             CHECK_AND(
#                 CHECK_IS(str),
#                 CHECK_NOT_EMPTY
#                 )
#             ),
#         None # TODO: REQUIRED on remote system
#         ),
    PalmConfig.Rule(
        'remote_username', ['host', 'remote', 'username'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        None # TODO: REQUIRED on remote system
        ),
    PalmConfig.Rule(
        'ssh_key', ['host', 'remote', 'ssh_key'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        None
        ),
    PalmConfig.Rule(
        'scp_port', ['host', 'remote', 'scp_port'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        None
        ),

    PalmConfig.Rule(
        'remote_jobcatalog', ['host', 'remote', 'logs'],
        FORMAT_PAIR,
        CHECK_ONE(
            CHECK_WITH( 'mode', CHECK_MATCH('local') ),
            CHECK_AND(
                CHECK_IS(str),
                CHECK_NOT_EMPTY
                )
            ),
        None  #TODO
        ),

    PalmConfig.Rule(
        'obs_frequency', ['host', 'remote', 'obs_frequency'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(int),
            CHECK_GT(0)
            ),
        3600
        ),

    PalmConfig.Rule(
        None, ['host', 'remote', 'cli', 'path'],
        FORMAT_NONE,
        CHECK_OR(
            CHECK_IS_NONE,
            CHECK_IS(str)
            ),
        '${path_base}/MAKE_DEPOSITORY_${configuration_identifier}/bin'
        ),

    # FIXME: with None, 'scp' or 'rsync' parameter
    #        'update' could determine the type of
    #        transfer
    #
    PalmConfig.Rule(
        None, ['host', 'remote', 'cli', 'update'],
        FORMAT_NONE,
        CHECK_OR(
            CHECK_IS_NONE,
            CHECK_IS(bool)
            ),
        True
        ),

    # build (make) ...
    #
    PalmConfig.Rule(
        None, ['build', 'make', 'command'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        'make'
        ),
    PalmConfig.Rule(
        'make_options', ['build', 'make', 'option'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        '-j$(nproc)'
        ),

    # build (cpp) ...
    #
    PalmConfig.Rule(
        None, ['build', 'cpp', 'command'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        'cpp'
        ),
    PalmConfig.Rule(
        'cpp_options', ['build', 'cpp', 'option'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        '-cpp -DMPI_REAL=MPI_DOUBLE_PRECISION -DMPI_2REAL=MPI_2DOUBLE_PRECISION -D__parallel -D__netcdf -D__fftw'
        ),

    # build (fortranSerial) ...
    #
    PalmConfig.Rule(
        'compiler_name_ser', ['build', 'fortranSerial', 'command'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        None # TODO: REQUIRED
        ),
    PalmConfig.Rule(
        None, ['build', 'fortranSerial', 'option'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        '-Ofast -ffree-line-length-none -I /usr/include -I /usr/local/include'
        ),

    # build (fortranParallel) ...
    #
    PalmConfig.Rule(
        'compiler_name', ['build', 'fortranParallel', 'command'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        None # TODO: REQUIRED
        ),
    PalmConfig.Rule(
        'compiler_options', ['build', 'fortranParallel', 'option'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        '-Ofast -ffree-line-length-none -I /usr/include -I /usr/local/include'
        ),

    # build (link) ...
    #
    PalmConfig.Rule(
        None, ['build', 'link', 'command'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        '${build_fortranParallel_command}'
        ),
    PalmConfig.Rule(
        'linker_options', ['build', 'link', 'option'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        '-L /usr/lib/x86_64-linux-gnu -L /usr/local/lib -lnetcdff -lfftw3'
        ),

    # commands (execute) ...
    #
    PalmConfig.Rule(
        'execute_command', ['commands', 'execute'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        '${process_mpi_command} ${process_mpi_option} palm'
        ),

    # commands (init) ...
    #
    PalmConfig.Rule(
        'login_init_cmd', ['commands', 'init'],
        FORMAT_INFIX,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        None
        ),

    # commands (module) ...
    #
    PalmConfig.Rule(
        'module_commands', ['commands', 'module'],
        FORMAT_INFIX,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        None
        ),

    # commands (input) ...
    #
    PalmConfig.Rule(
        'IC:', ['commands', 'input'],
        FORMAT_PREFIX,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        'ulimit -s unlimited'
        ),

    # commands (output) ...
    #
    PalmConfig.Rule(
        'OC:', ['commands', 'output'],
        FORMAT_PREFIX,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        (
        '[[ -f LIST_PROFIL_1D ]] && cat LIST_PROFIL_1D >> LIST_PROFILE\n'
        '[[ -f LIST_PROFIL    ]] && cat LIST_PROFIL    >> LIST_PROFILE'
        ),
        ),

    # commands (error) ...
    #
    PalmConfig.Rule(
        'EC: ', ['commands', 'error'],
        FORMAT_PREFIX,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        '[[ -f RUN_CONTROL ]] && cat RUN_CONTROL'
        ),

    # preprocess ...
    #
    PalmConfig.Rule(
        None, ['preprocess', 'command'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        '${parallelEnv_command}'
        ),
    PalmConfig.Rule(
        None, ['preprocess', 'option'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        ''
        ),

    # postprocess ...
    #
    PalmConfig.Rule(
        'process_combine_command', ['postprocess', 'command'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        '${parallelEnv_command}'
        ),
    PalmConfig.Rule(
        'process_combine_option', ['postprocess', 'option'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        (
        ''
        )
        ),
    PalmConfig.Rule(
        'execute_command_for_combine', None,
        FORMAT_PAIR,
        CHECK_NOTHING,
        '${process_combine_command} ${process_combine_option} combine_plot_fields.x'
        ),

    # parallelEnv ...
    #
    PalmConfig.Rule(
        'hostfile', ['parallelEnv', 'hostfile'],
        FORMAT_PAIR,
        CHECK_OR(
            CHECK_EQ('auto'),
            CHECK_IS_FILE
            ),
        'auto'
        ),
    PalmConfig.Rule(
        'process_mpi_command', ['parallelEnv', 'command'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        ''
        ),
    PalmConfig.Rule(
        'process_mpi_option', ['parallelEnv', 'option'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        ''
        ),
    PalmConfig.Rule(
        'cores', ['parallelEnv', 'cores'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(int),
            CHECK_GT(0)
            ),
        None
        ),
    PalmConfig.Rule(
        'threads_per_task', ['parallelEnv', 'openmpThreads'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(int),
            CHECK_GT(0)
            ),
        1
        ),
    PalmConfig.Rule(
        'tasks_per_node', ['parallelEnv', 'tasksPerNode'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(int),
            CHECK_GT(0)
            ),
        0
        ),
    PalmConfig.Rule(
        'maximum_parallel_io_streams', ['parallelEnv', 'maximumParallelIOstreams'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(int),
            CHECK_GT(0)
            ),
        None
        ),

    # batchEnv ...
    #
    PalmConfig.Rule(
        'submit_command', ['batchEnv', 'call'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        '${batchEnv_command} ${batchEnv_option}'
        ),

    PalmConfig.Rule(
        None, ['batchEnv', 'command'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        None # TODO: REQUIRED if batchEnv is used
        ),
    PalmConfig.Rule(
        None, ['batchEnv', 'option'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        None
        ),
    PalmConfig.Rule(
        'project_account', ['batchEnv', 'account'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        None
        ),
    PalmConfig.Rule(
        'defaultqueue', ['batchEnv', 'queue'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        None
        ),
    PalmConfig.Rule(
        'memory', ['batchEnv', 'memory'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(int),
            CHECK_GT(0)
            ),
        None
        ),

    # batchEnv (wallClockTime) ...
    #
    PalmConfig.Rule(
        'cpumax', ['batchEnv', 'cpumax'],
        FORMAT_PAIR,
        CHECK_AND(
            CHECK_IS(int),
            CHECK_GT(0)
            ),
        None
        ),

    # batchEnv (directives) ...
    #
    PalmConfig.Rule(
        'BD:', ['batchEnv', 'directives'],
        FORMAT_PREFIX,
        CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        (
        '#!/bin/bash\n'
        '#SBATCH --account={{project_account}}\n'
        '#SBATCH --job-name={{run_id}}\n'
        '#SBATCH --time={{cpu_hours}}:{{cpu_minutes}}:{{cpu_seconds}}\n'
        '#SBATCH --ntasks={{mpi_tasks}}\n'
        '#SBATCH --nodes={{nodes}}\n'
        '#SBATCH --ntasksPerNode={{tasksPerNode}}\n'
        '#SBATCH --partition={{queue}}\n'
        '#SBATCH --mem-per-cpu={{memory}}\n'
        '#SBATCH --output={{job_protocol_file}}\n'
        '#SBATCH --error={{job_protocol_file}}\n'
        '#SBATCH --mail-type=ALL\n'
        '#SBATCH --mail-user=<your-mail-address>\n'
        '#SBATCH --dependency=afterany:{{previous_job}}'
        )

        # possible variables to use in ${batchEnv_directives}:
        # {batchEnv_account} --> corresponds to 'project_account' within palmrun
        # run_id --> automatically declared within palmrun
        # {{cpu_hours}}:{{cpu_minutes}}:{{cpu_seconds}} --> calculated from {batchEnv_cpumax}, automatically declared within palmrun
        # mpi_tasks --> automatically declared within palmrun
        # nodes --> automatically declared within palmrun
        # {parallelEnv_tasksPerNode} --> corresponds to 'tasks_per_node'
        # {batchEnv_queue} --> corresponds to 'queue' adn 'defaultqueue' within palmrun
        # job_protocol_file --> automatically declared within palmrun
        # {batchEnv_memory} --> corresponds to 'memory' within palmrun
        # previous_job --> command-line option -W
        ),

    # define rules for internal parameter or programm arguments ...
    #
    PalmConfig.Rule( 'activation_string_list',
        validate=CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            )
        ),
    PalmConfig.Rule( 'create_batch_job',
        default=False
        ),
    PalmConfig.Rule( 'delete_temporary_catalog',
        default=True
        ),
    PalmConfig.Rule( 'configuration_identifier',
        validate=CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            ),
        default="default"
        ),
    PalmConfig.Rule( 'restart_run',
        default=False
        ),
    PalmConfig.Rule( 'create_jobfile_only',
        default=False
        ),
    PalmConfig.Rule( 'global_revision', #replaced with palm_version_string in git
        validate=CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            )
        ),
    PalmConfig.Rule( 'palm_version_string', #replaces global_revision in svn
        validate=CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            )
        ),
    PalmConfig.Rule( 'run_id_number',
        validate=CHECK_AND(
            CHECK_IS(str),
#            CHECK_MATCH(r"\d{5}", 'NNNNN')
            )
        ),
    PalmConfig.Rule( 'running_in_batch_mode',
        default=False
        ),
    PalmConfig.Rule( 'keep_data_from_previous_run',
        default=False
        ),
    PalmConfig.Rule( 'makefile',
        validate=CHECK_AND(
            CHECK_IS(str),
            CHECK_IS_FILE
            )
        ),
    PalmConfig.Rule( 'run_identifier',
        validate=CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            )
        ),
    PalmConfig.Rule( 'return_address',
        validate=CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            )
        ),
    PalmConfig.Rule( 'source_list',
        validate=CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            )
        ),
    PalmConfig.Rule( 'return_username',
        validate=CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            )
        ),
    PalmConfig.Rule( 'silent',
        default=False
        ),
    PalmConfig.Rule( 'use_existing_sources_folder',
        default=False
        ),
    PalmConfig.Rule( 'previous_job',
        validate=CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            )
        ),
    PalmConfig.Rule( 'do_trace',
        default=False
        ),
    PalmConfig.Rule( 'ocean_file_appendix',
        default=False
        ),
    PalmConfig.Rule( 'coupled_dist',
        validate=CHECK_AND(
            CHECK_IS(str),
            CHECK_NOT_EMPTY
            )
        ),
    PalmConfig.Rule( 'running_in_test_mode',
        default=False
        ),
    PalmConfig.Rule( 'combine_plot_fields',
        default=True
        ),
    PalmConfig.Rule( 'create_remote_batch_job',
        default=False
        ),

    # TODO: check need
    #
    # PalmConfig.Rule( 'link_local_input',            default=False     ),
    # PalmConfig.Rule( 'transfer_problems',           default=False     ),

    ]