if __name__ == '__main__': from os.path import dirname, abspath, join import sys sys.path.append( abspath( join( dirname(__file__), '..' ))) import collections import copy import io import os import unittest import palm.config from palm.PalmConfig import PalmConfig class ConfigTests(unittest.TestCase): __yaml__ = collections.OrderedDict([( 'path', collections.OrderedDict([ ('base', '' ), ('data', '' ), ('fast', 'fast' ), ('home', '$HOME/$path_fast' ), ('deps', '$path_fast/${path_fast}' ), ('dont', '$$path_home/deps' ) ]) )]) __yaml_str__ = \ "" \ "path:\n" \ " base: \n" \ " data: \n" \ " fast: fast\n" \ " home: $HOME/$path_fast\n" \ " deps: $path_fast/${path_fast}\n" \ " dont: $$path_home/deps\n" \ "" __yaml_rule__ = [ PalmConfig.Rule( 'base_directory', ['path', 'base'], palm.config.FORMAT_PAIR, palm.config.CHECK_IS_DEFINED ), PalmConfig.Rule( 'base_data', ['path', 'data'], palm.config.FORMAT_PAIR, None ), PalmConfig.Rule( 'fast_io_catalog', ['path', 'fast'], palm.config.FORMAT_PAIR, None ), PalmConfig.Rule( '', ['path', 'deps'], None, palm.config.DEFINE_CHECK( lambda value: '$' not in value, 'completly expanded' ) ), PalmConfig.Rule( 'configuration_identifier', default='default' ) ] __yaml_context__ = { 'HOME': '/home/user/path' } ################################################################# tests ### def test_yaml_name_empty(self): self.assertEqual( palm.config.yaml_name([]), '' ) self.assertEqual( palm.config.yaml_name(*[]), '' ) self.assertEqual( palm.config.yaml_name(), '' ) def test_yaml_name_default(self): self.assertEqual( palm.config.yaml_name(['path','base']), 'path_base' ) self.assertEqual( palm.config.yaml_name(*['path','base']), 'path_base' ) self.assertEqual( palm.config.yaml_name('path','base'), 'path_base' ) def test_yaml_name_dot(self): self.assertEqual( palm.config.yaml_name( ['path','base'], delim='.'), 'path.base' ) self.assertEqual( palm.config.yaml_name( *['path','base'], delim='.'), 'path.base' ) self.assertEqual( palm.config.yaml_name( 'path', 'base', delim='.'), 'path.base' ) def test_yaml_get_unknown(self): self.assertEqual( palm.config.yaml_get( self.__yaml__, ['path','???'] ), None ) self.assertEqual( palm.config.yaml_get( self.__yaml__, *['path','???'] ), None ) self.assertEqual( palm.config.yaml_get( self.__yaml__, 'path', '???' ), None ) def test_yaml_get_unknown_default(self): self.assertEqual( palm.config.yaml_get( self.__yaml__, *['path','???'], default='DEFAULT' ), 'DEFAULT' ) self.assertEqual( palm.config.yaml_get( self.__yaml__, ['path','???'], default='DEFAULT' ), 'DEFAULT' ) self.assertEqual( palm.config.yaml_get( self.__yaml__, 'path', '???', default='DEFAULT' ), 'DEFAULT' ) def test_yaml_get_known(self): self.assertEqual( palm.config.yaml_get( self.__yaml__, ['path','base'] ), '' ) self.assertEqual( palm.config.yaml_get( self.__yaml__, *['path','base'] ), '' ) self.assertEqual( palm.config.yaml_get( self.__yaml__, 'path','base' ), '' ) def test_yaml_get_known_subpath(self): self.assertEqual( palm.config.yaml_get( self.__yaml__, ['path'] ), self.__yaml__['path'] ) self.assertEqual( palm.config.yaml_get( self.__yaml__, *['path'] ), self.__yaml__['path'] ) self.assertEqual( palm.config.yaml_get( self.__yaml__, 'path' ), self.__yaml__['path'] ) def test_yaml_read_string(self): with io.StringIO( self.__yaml_str__) as yamlstream: self.assertEqual( palm.config.yaml_read( yamlstream ), self.__yaml__ ) def test_yaml_iterate_string(self): with io.StringIO( self.__yaml_str__) as yamlstream: iterator = palm.config.yaml_iterate(yamlstream) (path,value,group) = next(iterator) self.assertEqual( path, ['path', 'base'] ) self.assertEqual( value, '' ) (path,value,group) = next(iterator) self.assertEqual( path, ['path', 'data'] ) self.assertEqual( value, '' ) next(iterator) next(iterator) next(iterator) (path,value,group) = next(iterator) self.assertEqual( path, ['path', 'dont'] ) self.assertEqual( value, '$$path_home/deps' ) def test_yaml_expand_none(self): yaml = copy.deepcopy(self.__yaml__) rules = [] yaml, context = palm.config.yaml_expand( yaml, rules=rules, override=False, validate=False ) # NOTE: yaml was NOT expanded # self.assertEqual( palm.config.yaml_get( yaml, 'path', 'home' ), '$HOME/$path_fast' ) self.assertEqual( palm.config.yaml_get( yaml, 'path', 'deps' ), '$path_fast/${path_fast}' ) # NOTE: only known names are expanded # self.assertEqual( context.path_home, '$HOME/fast' ) self.assertEqual( context.path_deps, 'fast/fast' ) def test_yaml_expand_context(self): yaml = copy.deepcopy(self.__yaml__) rules = [] yaml, context = palm.config.yaml_expand( yaml, {}, # predefined self.__yaml_context__, # defaults rules=rules, override=False, validate=False ) # NOTE: yaml was NOT expanded # self.assertEqual( palm.config.yaml_get( yaml, 'path', 'home' ), '$HOME/$path_fast' ) self.assertEqual( palm.config.yaml_get( yaml, 'path', 'deps' ), '$path_fast/${path_fast}' ) self.assertEqual( context.path_home, '/home/user/path/fast' ) self.assertEqual( context.path_deps, 'fast/fast' ) # original context unchanged .., # self.assertFalse( 'path_home' in self.__yaml_context__ ) def test_yaml_to_palm_string(self): with io.StringIO( self.__yaml_str__) as yamlstream, io.StringIO() as palmstream: palm.config.yaml_to_palm( yamlstream, palmstream, rules=self.__yaml_rule__ ) self.assertRegex( palmstream.getvalue(), "# autogenerated by palm-cli: .*\n" \ "#\n" \ "%base_directory \s+\n" \ "%base_data \s+\n" \ "%fast_io_catalog\s+fast\n" \ ) # TODO: check when defaults are completed # # def test_default_rules(self): # config = PalmConfig(rules=palm.config.__PALM_RULES__) # def assertRule( yamlname, palmname, default ): # rule = config.rule(yamlname or palmname) # self.assertEqual( rule.pathname, yamlname ) # self.assertEqual( rule.name, palmname ) # self.assertEqual( rule.default, default ) # assertRule( 'path_base', 'base_directory', '$HOME/palm/current_version' ) # assertRule( 'path_runs', 'base_data', '${path_base}/runs' ) # assertRule( 'path_tmp', 'fast_io_catalog', '${path_base}/tmp' ) # assertRule( 'runfiles_input', 'runfiles_input', '${path_runs}/${run_identifier}/INPUT' ) # assertRule( 'runfiles_output', 'runfiles_output', '${path_runs}/${run_identifier}/OUTPUT' ) # assertRule( 'runfiles_logs', None, '${path_runs}/${run_identifier}/log_files' ) # assertRule( 'runfiles_monitoring', None, '${path_runs}/${run_identifier}/monitoring' ) # assertRule( 'runfiles_usercode', 'user_source_path', '${path_runs}/${run_identifier}/user_code' ) # assertRule( 'runfiles_restart', 'restart_data_path', '${path_runs}/${run_identifier}/restart' ) # assertRule( 'source_palm', 'source_path', '${path_base}/trunk/SOURCE' ) # assertRule( 'source_io-file-connection', None, '${path_base}/trunk/SCRIPTS/.palm.iofiles' ) # assertRule( 'host_local_ip', 'local_ip', None ) # assertRule( 'host_local_hostname', 'local_hostname', '$HOSTNAME' ) # assertRule( 'host_local_username', 'local_username', '$USER' ) # assertRule( 'host_local_path_base', None, '${path_base}' ) # assertRule( 'host_local_path_logs', 'local_jobcatalog', '${runfiles_logs}' ) # assertRule( 'host_local_path_tmp', None, '${path_tmp}' ) # assertRule( 'host_remote_ip', 'remote_ip', None ) # assertRule( 'host_remote_hostname', 'remote_hostname', None ) # assertRule( 'host_remote_username', 'remote_username', None ) # assertRule( 'host_remote_path_base', None, '${path_base}' ) # assertRule( 'host_remote_path_logs', 'remote_jobcatalog', '${runfiles_logs}' ) # assertRule( 'host_remote_path_tmp', None, '${path_tmp}' ) # assertRule( 'build_make_command', None, 'make' ) # assertRule( 'build_make_option', 'make_options', '-j$(nproc)' ) # assertRule( 'build_cpp_command', None, 'cpp' ) # assertRule( 'build_cpp_option', 'cpp_options', '-cpp -DMPI_REAL=MPI_DOUBLE_PRECISION -DMPI_2REAL=MPI_2DOUBLE_PRECISION -D__parallel -D__netcdf -D__fftw' ) # assertRule( 'build_fortran-serial_command', 'compiler_name_ser', None ) # assertRule( 'build_fortran-serial_option', None, '-Ofast -ffree-line-length-none -I /usr/include -I /usr/local/include' ) # assertRule( 'build_fortran-parallel_command', 'compiler_name', None ) # assertRule( 'build_fortran-parallel_option', 'compiler_options', '-Ofast -ffree-line-length-none -I /usr/include -I /usr/local/include' ) # assertRule( 'build_link_command', None, '${build_fortran-parallel_command}' ) # assertRule( 'build_link_option', 'linker_options', '-L /usr/lib/x86_64-linux-gnu -L /usr/local/lib -lnetcdff -lfftw3' ) # assertRule( 'commands_init', 'login_init_cmd', None ) # assertRule( 'commands_module', 'module_commands', None ) # assertRule( 'commands_input', 'IC:', 'ulimit -s unlimited' ) # assertRule( 'commands_output', 'OC:', ( # '[[ -f LIST_PROFIL_1D ]] && cat LIST_PROFIL_1D >> LIST_PROFILE\n' # '[[ -f LIST_PROFIL ]] && cat LIST_PROFIL >> LIST_PROFILE' # )) # assertRule( 'commands_error', 'EC: ', '[[ $locat = execution ]] && cat RUN_CONTROL' ) # assertRule( 'preprocess_command', None, '${parallel-env_command}' ) # assertRule( 'preprocess_option', None, None ) # assertRule( 'postprocess_command', 'process_combine_command', '${parallel-env_command}' ) # assertRule( 'postprocess_option', 'process_combine_option', '-n 1 --ntasks-per-node=1' ) # assertRule( None, 'execute_command_for_combine', '${process_combine_command} ${process_combine_option}' ) # assertRule( 'parallel-env_command', 'process_mpi_command', 'mpirun' ) # assertRule( 'parallel-env_option', 'process_mpi_option', None ) # assertRule( None, 'execute_command', '${process_mpi_command} ${process_mpi_option} palm' ) # assertRule( 'parallel-env_hostfile', 'hostfile', 'mpirun' ) # assertRule( 'parallel-env_cores', 'cores', None ) # assertRule( 'parallel-env_openmp-threads', 'threads_per_task', 1 ) # assertRule( 'parallel-env_tasks-per-node', 'tasks_per_node', None ) # assertRule( 'parallel-env_maximum-parallel-io-streams', 'maximum_parallel_io_streams', None ) # assertRule( 'batch-env_command', 'submit_command', None ) # assertRule( 'batch-env_option', None, None ) # assertRule( 'batch-env_account', 'project_account', None ) # assertRule( 'batch-env_queue', 'defaultqueue', None ) # assertRule( 'batch-env_memory', 'memory', 1000 ) # assertRule( 'batch-env_wall-clock-time_seconds', 'cpumax', None ) # assertRule( 'batch-env_wall-clock-time_timestring', None, None ) # assertRule( 'batch-env_directives', 'BD:', ( # for rule in config.rules: # print( # "assertRule( ", # "'"+rule.pathname+"'" if rule.pathname else None, # ", ", # "'"+rule.name+"'" if rule.name else None, # ", ", # "'"+rule.default+"'" if rule.default else None, # " )", # sep='' # ) def test_wall_clock_time_validation_pass(self): config = PalmConfig( rules=palm.config.__PALM_RULES__, override=True ) config['cpumax'] = 1234 del config['cpumax'] config['batch-env_wall-clock-time_timestring'] = '12:34:56' def test_wall_clock_time_validation_fails(self): config = PalmConfig( rules=palm.config.__PALM_RULES__, override=True ) self.assertRaises( AssertionError, config.__setitem__, 'cpumax', None ) self.assertRaises( AssertionError, config.__setitem__, 'cpumax', 'hello' ) self.assertRaises( AssertionError, config.__setitem__, 'cpumax', -1234 ) if __name__ == '__main__': unittest.main()