#!/usr/bin/env python3 # import argparse import os from os import path import stat import sys import shutil import time import math import random import datetime import subprocess if __name__ == '__main__': sys.path.append( path.abspath( path.join( path.dirname( path.realpath(__file__)), '..', '..' ))) import palm.config import palm.system import palm.cmd.build as palmbuild import palm.ioFiles as ioFiles LOG = palm.logger(__name__) def main( prog=None, argv=None ): # palm-cli [-h] run [] --bash # palm-cli [-h] run [] [] # result = 0 try: LOG.debug('start palmrun') opt, palm_argv, cli_argv = parse_option( prog, argv ) if opt.get('help'): build_argparse(prog).print_help() else: palm.init_logging(opt.get('log_append')) # process configuration file ... # yamlfile = palm.system.yaml_config_file(opt.get('configuration_identifier')) assert path.isfile(yamlfile), \ '\n +++ YAML configuration file: '+ \ '\n ' + yamlfile + \ '\n does not exist' yamldefs, palmdefs = palm.config.yaml_process( yamlfile, opt ) palmdefs.yamlfile = yamlfile palmdefs.program_name = 'palm-cli run' palmdefs.version = palm.system.version() palmdefs.palm_cli_options = ' '.join(cli_argv) palmdefs.palm_options = ' '.join(palm_argv) palmdefs.verbose = '' if palmdefs.silent else '-v' if '-O' in palmdefs.palm_options: palmdefs.use_openmp = True else: palmdefs.use_openmp = False LOG.debug( palmdefs.expand( 'cli options: ${palm_cli_options}' )) LOG.debug( palmdefs.expand( 'palm options: ${palm_options}' )) # ... and start running ... # if palmdefs.use_bash_script: # ... write to palm configuration file ... # palm.config.yaml_to_palm( palmdefs, palmdefs.config_file ) # ... and call script # result = palm.system.run( 'palmrun', palm_argv ) else: result = run(palmdefs, palm_argv) except KeyboardInterrupt: LOG.info('') LOG.info('+++ palm-cli run killed by "^C"') result = 2 except Exception as exception: if palm.traceback: LOG.error( exception, exc_info=True ) else: LOG.error( str(exception) ) result = 1 return result def run(defs, palm_argv): # useful helpers ... # def ASSERT( condition, message ): assert condition, defs.expand( message ) def EXP( message ): return defs.expand( message, keep_unknown=True ) def ENV( name, context=os.environ ): return defs.eval( name, context=context) def EXP_ENV( message,context=os.environ ): message = defs.expand( message ) result = defs.expand( message, context=context, keep_unknown=True ) return result def palm_shell( command, capture_output=False, ignore_error=False): return palm.system.shell( EXP(command), None, capture_output, ignore_error) def palm_chdir( path ): return palm.system.chdir( EXP(path), None ) def palm_mkdir( directory, name='', errmessage=''): if not path.isdir( directory ): try: os.makedirs(directory) if not defs.silent: LOG.info('') LOG.info('*** creating '+name+' directory:' ) LOG.info(' ' + directory ) except: LOG.error('') LOG.error('+++ can\'t create directory:' ) LOG.error(' ' + directory ) LOG.error(errmessage) raise def palm_ssh( command, capture_output=False, ignore_error=False ): return palm.system.shell( EXP('ssh -q ${ssh_keyopt} ${ssh_portopt} ${remote_username}@${remote_ip} \'"\'"\' %s \'"\'"\'' % command), capture_output=capture_output, ignore_error=ignore_error ) def palm_scp( local_file, remote_file, ignore_error=False ): if not defs.remote_env: defs.remote_env = dict( tuple(line.strip().split('=',1)) for line in palm_ssh('env', capture_output=True).splitlines()) return palm.system.shell( defs.expand( EXP('scp -q ${ssh_keyopt} ${portopt} %s ${remote_username}@${remote_ip}:%s' % (local_file,remote_file) ), defs.remote_env), ignore_error=ignore_error ) def analyseActivationStringList(string): activation_list=defs.activation_string_list.split() if string in activation_list: return '.true.' else: return '.false.' def readFirstLine(file): with open(file, 'r') as f: return f.readline().strip() def update_hostinfo(): try: import socket defs.hostname = socket.gethostname() defs.local_ip = socket.gethostbyname(defs.hostname) except: pass # REMEMBER CURRENT DIRECTORY # defs.path_current = os.getcwd() try: # CHECK, IF CONFIGURATION-FILE EXISTS ASSERT( defs.yamlfile, '+++ missing YAML-configuration, call palm-cli with "--pref "' ) defs.yamlfile = path.realpath(ENV('yamlfile')) ASSERT( path.isfile( defs.yamlfile ), '+++ YAML-configuration "${yamlfile}" does not exist' ) print('yamlfile: ', defs.yamlfile) # CHECK CONFIGURATION IDENTIFIER AND RUN IDENTIFIER ASSERT( defs.configuration_identifier, '+++ missing configuration identifier, call with "-c "' ) ASSERT( defs.run_identifier, '+++ missing run identifier, call with "-r "' ) defs.sources_for_run_catalog =EXP( 'SOURCES_FOR_RUN_${configuration_identifier}_${run_identifier}' ) LOG.info(EXP('\n ${program_name} ${version} \n will be executed. Please wait ...\n' )) # CHECK, IF THE ACTIVATION_STRING_LIST HAS BEEN GIVEN ASSERT( defs.activation_string_list, '+++ missing activation_string_list, call with "-a "' ) # SET VARIABLE (FORTRAN-SYTLE) TO ACTIVATE PALM BINARY OUTPUT FOR RESTARTS defs.write_binary = analyseActivationStringList('restart') # SET VARIABLE TO ACTIVATE WRITING OF SVF DATA defs.write_svf = analyseActivationStringList('svfout') # SET VARIABLE TO ACTIVATE READING OF SVF DATA defs.read_svf = analyseActivationStringList('svfin') # CHECK SETTING OF REQUIRED PARAMETERS ASSERT( defs.path_base, '+++ missing base directory in configuration file ${yamlfile}, please check "path:base" in that file' ) ASSERT( path.isdir(ENV('path_base')) , '+++ directory does not exist: ${path_base}' ) ASSERT( EXP('path_tmp'), '+++ missing fast_io_catalog in configuration file ${yamlfile}, please check "path:tmp" in that file' ) if not defs.running_in_batch_mode: defs.source_scripts = palm.system.scripts_dir() ASSERT( path.isdir(defs.source_scripts) , '+++ directory does not exist: ${source_scripts}' ) defs.source_palm = palm.system.source_dir() ASSERT( path.isdir(defs.source_palm) , '+++ directory does not exist: ${source_palm}' ) ASSERT( defs.compiler_name, '+++ missing compiler_name in configuration file ${yamlfile}, please check "build:fortranParallel:command" in that file' ) ASSERT( defs.compiler_name_ser, '+++ missing compiler name for non-parallel compilation in configuration file ${yamlfile}, please check "build:fortran-serial:command" in that file' ) ASSERT( defs.compiler_options, '+++ missing compiler options in configuration file ${yamlfile}, please check "build:fortranParallel:option" in that file' ) # CHECK SETTING OF COMMAND PARAMETER if defs.cores: # parallel run (MPI) or hybrid run (MPI and openMP) ASSERT( defs.parallelEnv_command, '+++ missing command for parallel execution of palm in configuration file, please check "parallelEnv:command" in that file' ) else: ASSERT(not ('mpi_tasks' in defs.parallelEnv_option), '+++ missing number of cores for parallel execution of palm, please set option -X ' ) if not defs.parallelEnv_command: if defs.use_openmp: LOG.info('+++ WARNING: Missing command for execution of palm in configuration file, please check block "parallelEnv" in that file!\n'\ +' Palm runs with openMP (option -O is set)!\n') else: LOG.info('+++ WARNING: Missing command for execution of palm in configuration file, please check block "parallelEnv" in that file!\n'\ +' Palm runs in serial mode!\n') if defs.hostfile != None: if defs.hostfile !='auto': ASSERT( path.isfile(ENV('hostfile')), '+++ hostfile ${hostfile} does not exist' ) #DETERMINE LOCAL HOSTNAME (defs.hostname) AND LOCAL IP ADDRESS (defs.local_ip) update_hostinfo() # DETERMINE THE CALL STATUS defs.create_remote_batch_job = False if defs.return_address != None: defs.running_on_remote = True else: if defs.remote_ip != None: defs.create_remote_batch_job = True ASSERT( defs.remote_jobcatalog, '+++ missing remote jobcatalog in configurations file ${yamlfile}, please check "host:remote:path:logs" in that file' ) ASSERT( not defs.create_batch_job, '+++ option -b must not be set because configuration file has been setup for running jobs on a remote host' ) ASSERT( defs.remote_username, '+++ missing remote username in configurations file ${yamlfile}, please check "host:remote:username" in that file' ) defs.running_on_remote = False # CHECK IF A LOCAL USER NAME HAS BEEN SET if defs.create_batch_job: ASSERT( defs.local_username, '+++ missing local username in configurations file ${yamlfile}, please check "host:local:username" in that file' ) # SET THE FILE CONNECTION FILE (CONFIGURATION FILE) defs.fileconnection_file =path.join(ENV('path_ioFileConnection'),'.palm.iofiles') # SET THE DEFAULT FILE CONNECTION FILE (WHICH IS IN THE REPOSITORY) if not defs.running_in_batch_mode: default_fileconnection_file = palm.system.default_iofiles() else: # IN BATCH MODE THE FILECONNECTION FILE CAN BE FOUND IN THE CURRENT FOLDER $tempdir default_fileconnection_file = '.palm.iofiles' defs.fileconnection_file = default_fileconnection_file # CHECK, IF FILE CONNECTION FILE EXISTS ASSERT( path.isfile( defs.fileconnection_file ), '+++ file connection file "' + defs.fileconnection_file + '" does not exist' ) LOG.info('\n Use yaml configuration file: '+ defs.yamlfile) LOG.info('\n Use file connection file: ' + defs.fileconnection_file) # CHECK, IF USER PROVIDES OWN FILE CONNECTION FILE if defs.fileconnection_file != default_fileconnection_file: # CHECK VERSION MISMATCH # RESTART RUNS DON'T REQUIRE A CHECK, BECAUSE THEY USE A COPY OF THAT # FILE WHICH HAS ALREADY BEEN CHECKED WITHIN THE INITAL PALMRUN CALL if defs.running_in_batch_mode == False and defs.restart_run == False: if readFirstLine(defs.fileconnection_file) != readFirstLine(default_fileconnection_file): LOG.warn(EXP('+++ WARNING: A file connection file has been found in your' \ +'\n working directory, but its revision does not match' \ +'\n the revision of the default version.' \ +'\n You may need to update your connection file' \ +'\n "${fileconnection_file}" !')) if not defs.silent: LOG.info ( '\n Reading the I / O files...') if defs.running_on_remote: remote_folder = path.join(ENV('path_tmp'), defs.sources_for_run_catalog) else: remote_folder ='' defs.dfin_pre, defs.dfout_pre = ioFiles.read_io2df(defs.fileconnection_file, defs.activation_string_list, create_remote_batch=defs.create_remote_batch_job, running_on_remote=defs.running_on_remote, remote_folder = remote_folder) # EVALUATE MODEL COUPLING FEATURES (OPTION -Y) defs.run_coupled_model = False if defs.coupled_dist != None: defs.run_coupled_model = True try: cores = [int(x) for x in defs.coupled_dist.split()] except: LOG.error('+++ wrong format in option -Y') LOG.error(' The parameter tells PALM how many cores shall be assigned to the atmosphere- and ocean-model, respectively.'\ '\n For example, in case of -X 64 -Y "16 48"" 16 cores are assigned to the atmosphere model, and 48 cores to the ocean model.') raise Exception defs.cores_atmos=cores[0] defs.cores_ocean=cores[1] ASSERT( sum(cores) == defs.cores, '+++ number of processors does not fit to specification by "-Y".'\ + '\n PEs (total) : ${cores}' \ + '\n PEs (atmosphere): $cores_atmos'\ + '\n PEs (ocean) : $cores_ocean' ) # IF I AM IN BATCH MODE, CHECK IF EXECUTABLE AND OTHER REQUIRED FILES # HAVE BEEN GENERATED BY PALMBUILD AND STORED IN THE SOURCES_FOR_RUN_... # FOLDER if defs.running_in_batch_mode: ASSERT( path.isdir(path.join(ENV('path_tmp'), defs.sources_for_run_catalog) ), '+++ directory ${fast_io_catalog}/${sources_for_run_catalog} is missing"'\ + '\n Please check the output of the palmrun-call"'\ + '\n that you did on your local host.' ) else: # CREATE THE SOURCES_FOR_RUN_... FOLDER, BUT NOT IF I AM PART OF AN # AUTOMATIC RESTART RUN # AUTOMATIC RESTART RUNS JUST ACCESS THE DIRECTORY CREATED BY THE INITIAL RUN if defs.restart_run == False: # COLLECT FILES TO BE COMPILED IN THE SOURCES_FOR_RUN_... FOLDER ON # THE LOCAL HOST ASSERT(path.isdir(ENV('source_palm')) , '+++ source path "'+ENV('source_palm')+'" on local host "' + defs.hostname +'" does not exist' ) defs.sfr_directory = path.join( ENV('base_directory'), defs.sources_for_run_catalog ) shutil.rmtree(defs.sfr_directory, ignore_errors=True) palm_mkdir(defs.sfr_directory, name='SOURCES_FOR_RUN') if defs.source_list == 'LM': # DETERMINE MODIFIED FILES OF THE ACTIVE GIT BRANCH path_save = os.getcwd() palm_chdir(ENV('source_palm')) defs.source_list=[] # CHECK, IF SRC-DIRECTORY IS UNDER GIT CONTROL ASSERT( path.isdir('../.git'), '+++ source directory "'+ENV('source_palm')+'"'\ + '\n is not under "git" control.'\ + '\n Plese do not use pamrun-option "-s LM".' ) # LIST ALL MODIFIED SOURCE CODE FILES palm_shell('git status --short > tmp_gitstatus') with open('tmp_gitstatus', 'r') as f: listfile_compile =[] lines = f.readlines() for line in lines: if line.strip().startswith(tuple(["M","A","?"])): columns=line.split() if columns[1].endswith(tuple([".py",".f90",".F90",".F",".f",".c"])): listfile_compile.append(columns[1]) os.remove('tmp_gitstatus') for file in listfile_compile: shutil.copy(file, defs.sfr_directory ) defs.source_list.append(path.basename(file)) palm_chdir(path_save) elif defs.source_list != None: # COPY FILES GIVEN BY OPTION -s TO DIRECTORY SOURCES_FOR_RUN_... source_list = defs.source_list.split() for file in source_list: ASSERT( path.dirname(file) =='', '+++ source code file must not contain a path' ) ASSERT( path.isfile( path.join(ENV('source_palm'),file) ), '+++ source code file "'+file+'" does not exist' ) shutil.copy( path.join(ENV('source_palm'),file) , defs.sfr_directory ) # CHECK, IF MAKEFILE EXISTS (OPTION -M) AND COPY IT TO THE SOURCES_FOR_RUN... DIRECTORY if defs.makefile == None: defs.makefile = path.join(ENV('source_palm'),'Makefile') ASSERT( path.isfile(defs.makefile), '+++ file "'+ENV('makefile')+ '" does not exist' ) shutil.copy(defs.makefile , defs.sfr_directory) if defs.user_source_path != None: if not path.isdir(ENV('user_source_path')): LOG.info('') LOG.info(EXP_ENV('*** INFORMATIVE: additional source code directory'\ +'\n "${user_source_path}"'\ +'\n does not exist or is not a directory.' \ +'\n No source code will be used from this directory!')) defs.user_source_path == None if not defs.silent: time.sleep(2) else: # COPY MAKEFILE FROM USER SOURCE PATH IF EXISTING if path.isfile(path.join(ENV('user_source_path'),'Makefile')): LOG.info('') LOG.info(EXP('*** user Makefile from directory'\ +'\n "'+ENV('user_source_path')+'" is used')) defs.makefile = path.join(ENV('user_source_path'),'Makefile') shutil.copy(defs.makefile, defs.sfr_directory) if not defs.silent: time.sleep(1) AddFilenames = [] for file in os.listdir(ENV('user_source_path')): if file.endswith(tuple([".f90",".F90",".F",".f",".c"])): AddFilenames.append(file) for file in AddFilenames: ASSERT( not path.isfile(path.join(defs.sfr_directory,file)), '\n +++ source code file "'+file+'" found in additional' \ +'\n source code directory ${user_source_path}' \ +'\n but was also given with option "-s" which means that it should be taken' \ +'\n from directory "${source_palm}.' ) makefile = path.join(defs.sfr_directory,'Makefile') #CHECK IF FILE IS CONTAINT IN MAKEFILE with open(makefile, 'r') as myMakefile: if file not in myMakefile.read(): LOG.error('+++ user file '+ file \ + '\n is not listed in Makefile \n') raise shutil.copy(path.join(ENV('user_source_path'),file) , defs.sfr_directory) try: # COPY CONFIGURATION FILES shutil.copy(defs.yamlfile, defs.sfr_directory) shutil.copy(defs.fileconnection_file, defs.sfr_directory) # COPY SHELLSCRIPTS shutil.copy(path.join(ENV('source_scripts'),'batch_scp'), defs.sfr_directory) except: LOG.error('') LOG.error('+++ failed to copy configuration files / shellscripts to \n'\ + defs.sfr_directory) raise Exception # GET THE SHA-1 CHECKSUM OF THE ACTIVE GIT BRANCH # (HANDED OVER TO RESTART-RUNS USING OPTION -G) if defs.palm_version_string == None: defs.palm_version_string = palm.system.version() # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS if defs.cores != None: # IN CASE OF PARALLEL EXECUTION, CHECK SOME SPECIFICATIONS CONCERNING PROCESSOR NUMBERS # CHECK, IF THE NUMBER OF CORES PER NODE HAS BEEN GIVEN UND IF IT IS AN # INTEGRAL DIVISOR OF THE TOTAL NUMBER OF CORES GIVEN BY OPTION -X if defs.tasks_per_node == 0: ASSERT( defs.create_batch_job == False and defs.create_remote_batch_job == False, '+++ option "-T" (tasks per node) is missing' \ + '\n set -T option or define tasks_per_node in the config file' ) defs.tasks_per_node = defs.cores ASSERT( defs.cores >= defs.tasks_per_node, '+++ tasks per node (-T) cannot exceed total number of coers (-X)'\ + '\n given value: -T ' + str(defs.tasks_per_node) + ' -X ' + str(defs.cores) ) defs.nodes = math.floor(defs.cores / (defs.tasks_per_node * defs.threads_per_task)) defs.mpi_tasks = math.floor(defs.cores / defs.threads_per_task) if defs.mpi_tasks == 0: defs.mpi_tasks = 1 ii = math.floor(defs.cores / defs.tasks_per_node) defs.remaining_cores = defs.cores - ii * defs.tasks_per_node if defs.remaining_cores > 0: LOG.info('') LOG.info(EXP('*** WARNING: tasks per node (option "-T") is not a integral'\ +'\n divisor of the total number of cores (option "-X")'\ +'\n values of this palmrun-call: "-T ${tasks_per_node}" "-X ${cores}"' \ +'\n One of the nodes is filled with ${remaining_cores} instead of ${tasks_per_node} tasks')) defs.nodes += 1 # SET DEFAULT VALUE FOR THE MAXIMUM NUMBER OF PARALLEL IO STREAMS if defs.maximum_parallel_io_streams == None: defs.optionW = False defs.maximum_parallel_io_streams = defs.cores # SET PORT NUMBER OPTION FOR CALLS OF SSH/SCP AND batch_scp SCRIPT defs.portopt = EXP('-P ${scp_port}') if defs.scp_port else '' defs.ssh_portopt = EXP('-p ${scp_port}') if defs.scp_port else '' # DETERMINE THE SSH-OPTION IN CASE THAT AN SSH-KEY IS EXPLICITLY GIVEN IN THE CONFIG-FILE defs.ssh_keyopt = EXP_ENV('-i $HOME/.ssh/${ssh_key}') if defs.ssh_key else '' defs.ssh_key = EXP_ENV('$HOME/.ssh/${ssh_key}') if defs.ssh_key else '' # CHECK IF QUEUE IS GIVEN (OPTION -q overrides SETTING in CONFIGURATION FILE) if defs.create_batch_job or defs.create_remote_batch_job: ASSERT( defs.defaultqueue, '+++ no default queue given in configuration file and no queue'\ + '\n given with option -q' ) #Renaming for compatibility to palmrun (bash) defs.queue = defs.defaultqueue # GENERATE FULL FILENAMES OF INPUT-FILES, INCLUDING THEIR PATH # CHECK, IF INPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST) LOG.info ( '\n Configuration of INPUT files') defs.dfin = ioFiles.config_inFiles(defs.dfin_pre, defs.run_identifier, defs.expand) # GENERATE FULL FILENAMES OF OUTPUT-FILES (WITHOUT $ OR ~), # CHECK, IF OUTPUT-FILES EXIST, AND DETERMINE HIGHEST CYCLE NUMBER (IF CYCLES EXIST), # OR, IN CASE THAT FILE DOES NOT EXIST, CHECK, IF IT CAN BE CREATED # THESE ACTIONS ARE NOT CARRIED OUT, IF FILES SHALL BE TRANSFERRED FROM THE REMOTE TO # THE LOCAL HOST (BECAUSE THEIR IS NO DIRECT ACCESS TO THE LOCAL DIRECTORIES FROM THE # REMOTE HOST) LOG.info('\n Configuration of OUTPUT files') #BUGFIX: source code moved after calling palm (reason: multiple output files in coupled run, nesting) # -> separate functions: ioFiles.config_outFiles_beforepalm(); ioFiles.config_outFiles_afterpalm() defs.dfout = ioFiles.config_outFiles_beforepalm(defs.dfout_pre, defs.expand, defs.running_on_remote, defs.run_identifier) # DETERMINE THE NAME OF PALMRUN'S TEMPORARY WORKING DIRECTORY if not defs.running_in_batch_mode: defs.run_id_number = random.randint(0,32767) defs.run_id = defs.run_identifier+'.'+str(defs.run_id_number) if defs.create_remote_batch_job: defs.remote_env = dict( tuple(line.strip().split('=',1)) for line in palm_ssh('env', capture_output=True).splitlines()) defs.tempdir =path.join( defs.expand(EXP('${path_tmp}'),defs.remote_env), defs.run_id) else: defs.tempdir =path.join( ENV('path_tmp'), defs.run_id) # FOR COMPATIBILITY REASONS WITH OLDER VERSIONS SET JOB_ID os.environ['job_id'] = defs.run_id else: # ALREADY RUNNING IN TEMPORARY WORKING DIRECTORY defs.tempdir = os.getcwd() # CHECK SETTINGS REQUIRED FOR BATCH JOBS if defs.create_batch_job or defs.create_remote_batch_job: # CHECK, IF JOB DIRECTIVES HAVE BEEN GIVEN IN CONFIGURATION FILE ASSERT( defs.batchEnv_directives, '+++ no batch directives found in configuration file' ) # CHECK IF CPUTIME IS GIVEN FOR JOB defs.cputime = defs.cpumax while not defs.cputime: try: answer = int(input('+++ cpu-time is undefined \n>>> Please type CPU-time in seconds as INTEGER: \n>>> ')) if answer > 0: defs.cputime = answer break except ValueError: LOG.info('Wrong input! Input must be an integer!') defs.cpumax = defs.cputime # CHECK THE MEMORY DEMAND while not defs.memory: try: answer = int(input('+++ memory demand is undefined \n>>> Please type memory in MByte per process as INTEGER: \n>>> ')) if answer > 0: defs.memory = answer break except ValueError: LOG.info('Wrong input! Input must be an integer!') # IN CASE OF REMOTE-JOBS CHECK, IF A USERNAME FOR THE REMOTE HOST IS GIVEN if defs.create_remote_batch_job and not defs.remote_username: while not defs.remote_username: answer = LOG.input(EXP('+++ username on remote host with IP "${remote_ip}" is undefined \n>>> Please type username: \n>>> ')) if answer: defs.remote_username = answer break else: if defs.running_in_batch_mode: defs.cputime = defs.cpumax else: defs.cputime = 10000000 defs.cpumax= defs.cputime # CALCULATE HOURS/MINUTES/SECONDS, E.G. FOR BATCH-DIRECTIVES defs.cpu_hours = math.floor(defs.cputime / 3600) defs.resttime = defs.cputime - defs.cpu_hours * 3600 defs.cpu_minutes = math.floor(defs.resttime / 60) defs.cpu_seconds = defs.resttime - defs.cpu_minutes * 60 # OUTPUT OF THE PALMRUN-HEADER print_summary(defs) # OUTPUT OF FILE CONNECTIONS IN CASE OF TRACEBACK if defs.do_trace: LOG.info('\n\n >>> INPUT-file assignments:\n') for i in defs.dfin.index: LOG.info(defs.dfin.loc[i,'local'] + ': '+EXP_ENV(defs.dfin.loc[i,'absname'])) LOG.info('\n\n >>> OUTPUT-file assignments:\n') for i in defs.dfout.index: LOG.info(defs.dfout.loc[i,'local'] + ': '+EXP_ENV(defs.dfout.loc[i,'path'])) LOG.info('\n\n >>> INPUT-commands:\n') LOG.info(defs.commands_input) LOG.info('\n\n >>> OUTPUT-commands:\n') LOG.info(defs.commands_output) # QUERY FOR CONTINUE if not defs.silent and not defs.running_in_batch_mode: while True: LOG.info('>>> Everything o.k. (y/n) ?') answer = input().lower() if answer == 'n': LOG.info('\n +++ palm-ci run aborted\n') return if answer == 'c' or answer == 'continue' or answer == 'y' or answer == 'yes' or answer == 's': break if defs.create_batch_job or defs.create_remote_batch_job: LOG.info('\n*** batch-job will be created and submitted') else: LOG.info('\n*** PALMRUN will now continue to execute on this machine') # PROVIDE FILES TO EXECUTE PALM AND CREATE THE EXECUTABLE if not defs.restart_run and not defs.running_in_batch_mode: if defs.create_remote_batch_job: LOG.info('\n\n*** creating executable and other sources for the remote host') else: LOG.info('\n\n*** creating executable and other sources for the local host') # FIRST CHECK, IF A MAKE DEPOSITORY EXISTS, AND IF NOT, ASK THE USER IF # IT SHALL BE CREATED ask_for_make_depository = False if defs.create_remote_batch_job: # CHECK FOR MAKE_DEPOSITORY ON THE REMOTE HOST defs.make_depository = path.join( defs.expand(EXP('${path_base}'),defs.remote_env),EXP('MAKE_DEPOSITORY_${configuration_identifier}')) command = EXP('[[ -d ${make_depository} ]] && echo depository available || echo depository not found' ' 2>&1;' ' exit $?\n') status = palm_ssh(command, capture_output = True) ask_for_make_depository = False if status == 'depository not found': LOG.info('\n\n+++ make depository "'+ defs.make_depository + '"\n'\ + ' on remote host not found!') ask_for_make_depository = True else: LOG.info('\n*** make depository "'+ defs.make_depository + '"\n'\ + ' on remote host available!') else: # CHECK FOR MAKE_DEPOSITORY ON THE LOCAL HOST defs.make_depository = path.join( ENV('base_directory'), EXP('MAKE_DEPOSITORY_${configuration_identifier}')) if not path.isdir(defs.make_depository): LOG.info('\n\n+++ make depository "'+ defs.make_depository + '"\n'\ + ' on local host not found!') ask_for_make_depository = True if ask_for_make_depository: while True: LOG.info('>>> Create a new one (y/n) ?') answer = input().lower() if answer == 'n': LOG.info('\n +++ palm-ci run aborted\n') return if answer == 'c' or answer == 'continue' or answer == 'y' or answer == 'yes' or answer == 's': break if defs.do_trace: result = palmbuild.main(None, ['-c',defs.configuration_identifier]) else: result = palmbuild.main(None, ['-v','-c',defs.configuration_identifier,'--no-summary']) if result == 0: LOG.info('\n*** now continue with creating executable and other sources') else: LOG.error('\n+++ error while compiling for the MAKE_DEPOSITORY') raise Exception # NOW CREATE THE SOURCES_FOR_RUN FOLDER if defs.use_existing_sources_folder: result = palmbuild.main(None, ['-V','-v', '-c',defs.configuration_identifier,'-r',defs.run_identifier,'--no-summary']) else: result = palmbuild.main(None, ['-v', '-c',defs.configuration_identifier,'-r',defs.run_identifier,'--no-summary']) if result == 0: LOG.info('\n*** executable and other sources created') shutil.rmtree(defs.sfr_directory, ignore_errors=True) else: shutil.rmtree(defs.sfr_directory, ignore_errors=True) LOG.error('\n+++ error while creating executable and/or other sources') raise Exception # WHEN CREATING A REMOTE BATCH JOB, THOSE INPUT FILES WITH TRANSFER-ATTRIBUT # WILL BE COPIED TO THE REMOTE HOST if defs.create_remote_batch_job: counter = 0 for i in defs.dfin.index: if defs.dfin.loc[i,'action'] == 'tr': #scp -q ${ssh_keyopt} ${portopt} ${localfile} ${remote_username}@${remote_ip}:${fast_io_catalog}/${sources_for_run_catalog}/${file}' localfile =EXP_ENV(path.join(defs.dfin.loc[i,'path'], defs.dfin.loc[i,'frel']) ) defs.file = EXP_ENV(defs.dfin.loc[i,'frel']) remote_file = EXP('${fast_io_catalog}/${sources_for_run_catalog}/${file}') palm_scp(localfile, remote_file) counter += 1 if counter > 0: LOG.info("\n*** input files have been copied to the remote host") # NOW PERFORM THOSE ACTIONS REQUIRED TO EXECUTE THE PROGRAM (PALM) ON THIS MACHINE # (COMPILING/LINKING, EXECUTING, COPYING I/O FILES) if not defs.create_batch_job and not defs.create_remote_batch_job: # CHANGE TO THE TEMPORARY WORKING DIRECTORY if not defs.running_in_batch_mode: # CREATE THE DIRECTORY palm_mkdir(defs.tempdir, name='TEMPORORY', errmessage=' Check setting of variable fast_io_catalog in your config file.') #chmod: go+rx os.chmod(defs.tempdir, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) palm_chdir(defs.tempdir) LOG.info('\n*** changed to temporary directory') else: LOG.info('\n*** running in temporary directory:\n '+defs.tempdir) # PROVIDE THE INPUT FILES # LOOP OVER ALL ACTIVATED FILES (LISTED IN THE CONFIGURATION FILE) #TODO (!?) link_local_input = true: parameter not implemented yet ioFiles.provide_inFiles(defs.dfin, defs.expand, defs.cores, defs.running_on_remote) # NOW COPY FILES (*.f90, *.o, config files, etc.) FROM SOURCES_FOR_RUN_... TO THE TEMPORARY # WORKING DIRECTORY sfr_source = path.join(ENV('path_tmp'), defs.sources_for_run_catalog) for file in os.listdir(sfr_source): # BUGFIX copy palm-cli files from subdirectory of sfrdir to subdirectory of tempdir try: shutil.copy(path.join(sfr_source,file), defs.tempdir) except: shutil.copytree(path.join(sfr_source, file), path.join(defs.tempdir, file)) # EXECUTE INPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE if defs.commands_input: LOG.info('\n\n*** execution of INPUT-commands:') LOG.info('-------------------------------------') LOG.info(defs.commands_input) LOG.info('-------------------------------------') palm_shell(defs.commands_input, ignore_error=True) # CHECK IF THE PROGRESS BAR NEEDS TO BE DISABLED if defs.running_in_batch_mode or defs.running_in_test_mode: defs.progress_bar_disabled = '.true.' else: defs.progress_bar_disabled = '.false.' # CREATE THE NAMELIST-FILE WITH VALUES OF ENVIRONMENT-VARIABLES REQUIRED BY PALM # (FILE ENVPAR WILL BE READ BY PALM) with open('ENVPAR','w') as envpar: envpar.write(EXP(' &envpar run_identifier = "${run_identifier}", host = "${configuration_identifier}",\n')) envpar.write(EXP(' write_svf = ${write_svf}, write_binary = ${write_binary}, \n')) envpar.write(EXP(' read_svf = ${read_svf}, tasks_per_node = $tasks_per_node,\n')) envpar.write(EXP(' maximum_parallel_io_streams = $maximum_parallel_io_streams,\n')) envpar.write(EXP(' maximum_cpu_time_allowed = ${cpumax}.,\n')) envpar.write(EXP(' version_string = "${palm_version_string}",\n')) envpar.write(EXP(' progress_bar_disabled = ${progress_bar_disabled} / \n')) # STARTING THE EXECUTABLE LOG.info('\n\n*** execution starts in directory' \ + '\n '+os.getcwd() ) LOG.info ('-------------------------------------------') #ADD TEMPORARY DIRECTORY TO PATH VARIABLE (ENVIRONMENT) os.environ['PATH'] += os.pathsep + defs.tempdir #GENERATE EXECUTION COMMANDS AND REPLACE PARAMETERS IN THE EXECUTION COMMAND WITH REAL VALUES # #THE FOLLOWING STEPS ARE ALREADY PERFORMED IN config.py AS RULES (-> THEREFORE COMMENTED) # defs.execute_command = EXP('${parallelEnv_command} ${parallelEnv_option} ./palm') # # # EXECUTION COMMAND FOR COMBINE_PLOT_FIELDS # if defs.postparallelEnv_command != '': # defs.execute_command_for_combine = EXP('${postprocess_command} ${postprocess_option} ./combine_plot_fields.x') #work arround: Compatibility with palmrun (bash) syntax defs.execute_command = defs.execute_command.replace('{{','${') defs.execute_command = defs.execute_command.replace('}}','}') defs.execute_command_for_combine = defs.execute_command_for_combine.replace('{{mpi_tasks}}','1') defs.execute_command_for_combine = defs.execute_command_for_combine.replace('{{tasks_per_node}}', '1') # PROVIDE A HOSTFILE, IF REQUIRED if defs.cores != None: if defs.hostfile !='': if defs.hostfile == "auto": # CREATE A NEW HOSTFILE defs.hostfile = 'hostfile' with open(defs.hostfile, 'w') as hostfile: hostfile.write((defs.hostname+' \n')*defs.mpi_tasks) else: shutil.copy(defs.hostfile,'hostfile') command = 'head -n' + str(defs.mpi_tasks) + ' '+ENV('hostfile') valInfo = palm_shell( command, capture_output=True ) LOG.info('\n*** running on: '+valInfo.replace('\n','')) defs.execute_command = EXP_ENV(defs.execute_command) defs.execute_command_for_combine = EXP_ENV(defs.execute_command_for_combine) # SET THE NUMBER OF OPENMP-THREADS if defs.use_openmp: os.environ['OMP_NUM_THREADS'] = str(defs.threads_per_task) LOG.info('\n*** number of OpenMP threads per MPI-task: '+str(defs.threads_per_task)) # PROVIDE DATA FOR ATMOSPHERE OCEAN COUPLING cs_file = open('coupling_steering','w') if not defs.run_coupled_model: if defs.ocean_file_appendix: cs_file.write('precursor_ocean') else: cs_file.write('precursor_atmos') else: iia = math.floor(defs.cores_atmos / defs.threads_per_task) iio = math.floor(defs.cores_ocean / defs.threads_per_task) LOG.info('\n coupled run (' + str(iia) + ' atmosphere, ' + str(iio) + ' ocean)') cs_file.write('coupled_run ' + str(iia) + ' ' + str(iio)) cs_file.close() LOG.info('\n*** execute command:') LOG.info(' "'+defs.execute_command + '"\n\n') if defs.progress_bar_disabled == '.true.': command = defs.execute_command + ' < coupling_steering &> >(grep -v --line-buffered -e "^STOP 1$" -e "^1$" &> >(tee STDOUT) )' else: command = defs.execute_command + " < coupling_steering &> >(tee STDOUT)" palm_shell(command) LOG.info ('-------------------------------------------') LOG.info ('*** execution finished\n') # CALL OF combine_plot_fields IN ORDER TO MERGE SINGLE FILES WRITTEN # BY EACH CORE INTO ONE FILE if not path.isfile('combine_plot_fields.x'): LOG.info('\n\n+++ WARNING: no combine_plot_fields found') LOG.info(' 2d- and/or 3d-data may be incomplete!') LOG.info(' Your previous palmbuild may have failed. Please check.\n') elif defs.combine_plot_fields: LOG.info('\n\n*** post-processing: now executing "' + defs.execute_command_for_combine+'" ...') palm_shell(defs.execute_command_for_combine) else: # TEMPORARY SOLUTION TO SKIP combine_plot_fields. THIS IS REQUIRED IN CASE OF HUGE AMOUNT OF # DATA OUTPUT LOG.info('\n\n*** post-processing: skipping combine_plot_fields (-Z option set) ...') # EXECUTE OUTPUT-COMMANDS GIVEN IN THE CONFIGURATION FILE LOG.info('\n\n*** execution of OUTPUT-commands:') LOG.info ('-------------------------------------------') palm_shell("${commands_output}", ignore_error=True) LOG.info ('-------------------------------------------') #DEFINE YAMLFILE FOR SCP TRANSFER if defs.running_on_remote: remote_jobcatalog_env=ENV('remote_jobcatalog') defs.scp_yamlfile=path.join(remote_jobcatalog_env,'scp_jobfile.'+defs.configuration_identifier) + '_'+defs.run_identifier+ '.' + str(defs.run_id_number) + '.yml' else: defs.scp_yamlfile = '' #BUGFIX: source code moved after calling palm (reason: multiple output files in coupled run, nesting) # CONFIGURATION OF MULTIPLE FILES: FOR EACH BASENAME ENDING CREATE AN ENTRY IN THE FINAL OUTPUT FILE LIST defs.dfout_final = ioFiles.config_outFiles_afterpalm(defs.dfout, defs.expand, defs.running_on_remote) # IN A FIRST PASS, ADD ADDITIONAL OUTPUT FILE CONNECTIONS IN CASE OF # WILDCARDS ioFiles.provide_outFiles(defs.dfout_final, defs.expand, defs.cores, defs.run_identifier, defs.configuration_identifier, defs.run_id_number, defs.running_on_remote, defs.scp_yamlfile) # IF REQUIRED, START A RESTART-JOB # FILE CONTINUE_RUN MUST HAVE BEEN CREATED BY THE EXECUTABLE (PALM) if path.isfile('CONTINUE_RUN'): palm_argv_tmp = ['"'+arg+'"' if (not '-' in arg and not str.isdigit(arg)) else arg for arg in palm_argv] # DELETE SOME OPTIONS WITHOUT VALUE palm_argv = [arg for arg in palm_argv_tmp if arg not in ['-F','-j','-z']] # DELETE SOME OPTIONS WITH VALUE search = ['-i','-R','-s','-W'] for option in search: if option in palm_argv: index = palm_argv.index(option) del palm_argv[index + 1] del palm_argv[index] defs.palm_options = ' '.join(palm_argv) # ADD RESTART-OPTIONS TO THE PALMRUN-CALL (IF THEY ARE NOT USED ALREADY): # -C TELLS PALMRUN THAT IT IS A RESTART-RUN # -v SILENT MODE WITHOUT INTERACTIVE QUERIES # -b START A BATCH JOB if not '-C' in defs.palm_options: defs.palm_options = defs.palm_options + ' -C' if not '-v' in defs.palm_options: defs.palm_options = defs.palm_options + ' -v' if not '-b' in defs.palm_options and not defs.running_on_remote: defs.palm_options = defs.palm_options + ' -b' # REPLACE SVFOUT IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a) # SO THAT RESTARTS USE SVF DATA CREATED BY THE INITIAL RUN if analyseActivationStringList('svfout'): defs.palm_options = defs.palm_options.replace('svfout','svfin') # REPLACE THE HASH IN THE ACTIVATION STRINGS (GIVEN WITH OPTION -a) # SO THAT RESTARTS ACCESS DIFFERENT FILES THAN THE INITIAL RUN defs.palm_options = defs.palm_options.replace('#','r') #REBUILD THE PALMRUN-COMMAND STRING if defs.running_on_remote: #LOCAL_YAML = os.getenv('LOCAL_YAML') prc = ' '.join(['palm-cli run', '--log-append', defs.palm_options, defs.palm_cli_options]) else: prc = ' '.join(['palm-cli run', '--log-append', defs.palm_options, defs.palm_cli_options]) # START THE RESTART-JOB LOG.info(EXP('*** initiating restart-run on ${local_ip} using command:')) LOG.info(' "'+ prc+'"') LOG.info(' ----------------------------------------------------------------------------') if defs.running_on_remote == True: LOG.info('*** Initiate restart-runs!') defs.remote_jobcatalog_env=ENV('remote_jobcatalog') defs.run_id = path.basename(defs.tempdir) restart_file = EXP('${remote_jobcatalog_env}/palm_restart_${configuration_identifier}_${run_id}') LOCAL_PALMCLI_PATH = os.getenv('LOCAL_PALMCLI_PATH') LOCAL_PWD = os.getenv('LOCAL_PWD') #this file will be fetched by watchdog_remote script with open(restart_file,'w') as rf: rf.write('PATH=$PATH:'+LOCAL_PALMCLI_PATH + '\n') rf.write('cd '+LOCAL_PWD + '\n') rf.write(prc +'\n') else: # START THE RESTART JOB ON THE LOCAL HOST command = prc +' | tee palmrun_restart.log' palm_shell(command) time.sleep(3) # CHECK, IF RESTART JOB HAS BEEN STARTED with open('palmrun_restart.log') as f: content = f.read() if 'palm-cli run crashed' in content: LOG.info(' ----------------------------------------------------------------------------') LOG.info('+++ creating restart run failed') os.remove('palmrun_restart.log') raise Exception elif 'palm-cli run finished' in content: LOG.info(' ----------------------------------------------------------------------------') LOG.info('+++ creating restart run failed, probably due to network problems') os.remove('palmrun_restart.log') raise Exception else: LOG.info(' ----------------------------------------------------------------------------') LOG.info('*** restart run initiated') os.remove('palmrun_restart.log') # DELETE INPUT-(RESTART)FILES, WHICH HAVE BEEN FETCHED FROM THE TEMPORARY DATA # DIRECTORY, BECAUSE THEY ARE NOT REQUIRED BY THE RESTART-JOB. # THIS IS DONE IN ORDER TO AVOID EXCEEDING DISC QUOTAS OR DISC SPACE (RESTART-FILES # MAY BE VERY HUGE) for i in defs.dfin.index: if defs.dfin.loc[i,'gottmp'] and defs.keep_data_from_previous_run == False: os.remove( defs.dfin.loc[i,'absname']) else: # PREPARING ACTIONS, # IF A BATCH-JOB IS TO BE GENERATED AND TO BE STARTED ON A LOCAL OR REMOTE-MACHINE # BUILD THE PALMRUN-COMMAND TO BE CALLED IN THE BATCH-JOB defs.username_env = ENV('local_username') defs.palmrun_com = EXP('palm-cli run -r ${run_identifier} -c ${configuration_identifier} -m ${memory} -t ${cpumax} -q "${queue}" -i "${run_id_number}" -U "${username_env}"') if defs.activation_string_list: defs.palmrun_com = defs.palmrun_com + EXP(' -a "${activation_string_list}"') #if defs.palm_version_string: # defs.palmrun_com = defs.palmrun_com + EXP(' -G "${palm_version_string}"') if defs.keep_data_from_previous_run: defs.palmrun_com = defs.palmrun_com + ' -k' if defs.do_trace: defs.palmrun_com = defs.palmrun_com + ' -x' if defs.cores: defs.palmrun_com = defs.palmrun_com + EXP(' -X ${cores}') if defs.use_openmp: defs.palmrun_com = defs.palmrun_com + EXP(' -O ${threads_per_task}') if defs.tasks_per_node: defs.palmrun_com = defs.palmrun_com + EXP(' -T ${tasks_per_node}') if not defs.delete_temporary_catalog: defs.palmrun_com = defs.palmrun_com + ' -B' if defs.ocean_file_appendix: defs.palmrun_com = defs.palmrun_com + ' -y' if defs.run_coupled_model: defs.palmrun_com = defs.palmrun_com + EXP(' -Y ${coupled_dist}') if not defs.combine_plot_fields: defs.palmrun_com = defs.palmrun_com + ' -Z' if defs.optionW: defs.palmrun_com = defs.palmrun_com + EXP(' -w ${maximum_parallel_io_streams}') if defs.project_account: defs.palmrun_com = defs.palmrun_com + EXP(' -A ${project_account}') defs.local_jobcatalog_env=ENV('local_jobcatalog') if defs.create_remote_batch_job: #DETERMINE THE FULL PATH FOR THE FAST IO CATALOG defs.fast_io_catalog_env=ENV('fast_io_catalog', defs.remote_env) defs.remote_username_env = ENV('remote_username') defs.palmrun_com=defs.palmrun_com + EXP(' -j -u ${remote_username_env} -R ${local_ip}') if defs.do_trace: LOG.info('*** PALMRUN-command on remote host:\n '+defs.palmrun_com) # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE REMOTE HOST defs.remote_jobcatalog_env=ENV('remote_jobcatalog', defs.remote_env) defs.job_protocol_file_remote=EXP('${remote_jobcatalog_env}/${configuration_identifier}_${run_id}') defs.job_protocol_file=defs.job_protocol_file_remote elif defs.create_batch_job: #DETERMINE THE FULL PATH FOR THE FAST IO CATALOG defs.fast_io_catalog_env=ENV('fast_io_catalog') defs.palmrun_com = defs.palmrun_com + ' -j' if defs.do_trace: LOG.info('*** PALMRUN-command on local host:\n '+defs.palmrun_com) # DETERMINE THE FULL PATHS FOR THE JOB PROTOCOL FILES ON THE LOCAL HOST defs.job_protocol_file_local=EXP('${local_jobcatalog_env}/${configuration_identifier}_${run_id}') defs.job_protocol_file=defs.job_protocol_file_local # BUILD THE JOB-SCRIPTS ON FILE jobfile defs.jobfile=EXP('jobfile.${run_id_number}') with open(defs.jobfile,'w') as jf: # FIRST CREATE THE BATCH DIRECTIVES array =defs.batchEnv_directives.split('\n') for line in array: newline= line.replace('{{','${') defs.newline = newline.replace('}}','}') jf.write(EXP_ENV(defs.newline)+'\n') jf.write('\n') # ACTIVATE ERROR-TRACEBACK if defs.do_trace: jf.write('set -x\n') else: jf.write('set +vx\n') # INITIALIZE THE ENVIRONMENT AND LOAD MODULES if defs.login_init_cmd: jf.write(EXP('${login_init_cmd}\n')) if defs.module_commands: jf.write(EXP('${module_commands}\n')) # CREATE TEMPORARY DIRECTORY AND SWITCH TO IT if defs.create_remote_batch_job: jf.write(EXP('mkdir ${tempdir}\n')) jf.write('if [[ $? != 0 ]]\n') jf.write('then\n') jf.write(' echo \"+++ temporary working directory cannot be created.\" \n') jf.write(' echo \" Check setting of variable fast_io_catalog in your config file.\" \n') jf.write(' exit \n') jf.write('fi\n') jf.write(EXP('chmod go+rx ${tempdir}\n')) else: # DIRECTORY FOR LOCAL BATCH JOBS IS CREATED NOW, DUE TO A # REQUIREMENT OF THE GRID ENGINE BATCH SYSTEM (WORKING DIR IS GIVEN IN # BATCH DIRECTIVE -wd AND MUST ALREADY EXIST WHEN THE JOB IS SUBMITTED) palm_mkdir( defs.tempdir, name='TEMPORORY', errmessage=' Check setting of variable fast_io_catalog in your config file.') #chmod: go+rx os.chmod(defs.tempdir, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) jf.write(EXP('cd ${tempdir}\n')) jf.write('if [[ $? != 0 ]]\n') jf.write('then\n') jf.write(' echo \"+++ TEMPORARY directory does not exist:\" \n') jf.write(EXP(' echo \" ${tempdir}" \n')) jf.write(' exit \n') jf.write('fi\n') jf.write(EXP('export tempdir=${tempdir}\n')) jf.write(EXP('cp ${fast_io_catalog_env}/${sources_for_run_catalog}/.[!.]* .\n')) # BUGFIX: palm-cli is copied to bin/ --> path variable must point to bin/ jf.write(EXP('export PATH=.:${fast_io_catalog_env}/${sources_for_run_catalog}/bin:$PATH\n')) jf.write('export execute_palmrun=true\n') # PROVIDE NAME OF THE CURRENT WORKING-DIRECTORY ON THE LOCAL MACHINE (FROM WHERE THE JOB IS # STARTED) BY SETTING AN ENVIRONMENT-VARIABLE. THIS INFORMATION IS USED IN THE JOB BY PALMRUN # IN CASE THAT RESTART-RUNS HAVE TO BE GENERATED defs.working_directory = os.getcwd() jf.write(EXP('LOCAL_PWD=${working_directory}\n')) jf.write('export LOCAL_PWD\n') #PROVIDE FULL PATH OF CURRENT YAML FILE FOR THE SAME REASON #jf.write(EXP('LOCAL_YAML=${yamlfile}\n')) #jf.write('export LOCAL_YAML\n') # PROVIDE THE PATH OF THE LOCAL PALM-CLI RUN-SCRIPT FOR THE SAME REASON defs.palmcli_dir = palm.system.cli_dir() jf.write(EXP('LOCAL_PALMCLI_PATH=${palmcli_dir}\n')) jf.write('export LOCAL_PALMCLI_PATH\n') # CALL PALMRUN WITHIN THE JOB # AS FINAL ACTION, REMOVE THE TEMPORARY DIRECTORY CREATED AT THE BEGINNING OF THE JOB jf.write('set -x\n') jf.write(EXP('[[ $execute_palmrun = true ]] && ${palmrun_com}\n')) # TRANSFER JOBFILE TO THE TARGET HOST if not defs.create_jobfile_only: # CREATE LOCAL JOB FOLDER if not path.isdir(defs.local_jobcatalog_env): LOG.info(EXP('*** local jobcatalog "${local_jobcatalog_env}" does not exist and will be created now')) palm_mkdir( defs.local_jobcatalog_env, name='LOCAL_JOBCATALOG') if defs.create_remote_batch_job: LOG.info('\n*** transfer of job to remote host via scp') # CREATE REMOTE JOB FOLDER, IF IT DOES NOT EXIST command = EXP('if [[ ! -d ${remote_jobcatalog} ]]\n' ' then\n' ' mkdir -p ${remote_jobcatalog}\n' ' fi\n') palm_ssh(command) # COPY THE JOB FILE TO REMOTE HOST if defs.do_trace: LOG.info(EXP('scp ${ssh_keyopt} ${portopt} ${jobfile} ${remote_username}@${remote_ip}:${remote_jobcatalog}/${configuration_identifier}_${run_id}')) remote_file = EXP('${remote_jobcatalog}/${configuration_identifier}_${run_id}') palm_scp( defs.jobfile, remote_file) # SUBMIT THE JOB LOG.info('\n*** submit the job (output of submit command,'\ '\n e.g. the job-id given by the batch-system, may follow)') command = EXP(' cd ${remote_jobcatalog};' ' $submit_command ${configuration_identifier}_${run_id};' ' rm ${configuration_identifier}_${run_id}; exit $?\n ') if defs.do_trace: LOG.info(command) #SUBMIT COMMAND palm_ssh(command) #DEFINE SOME FILENAMES WITH FULL RESOLVED PATHES NECESSARY FOR WATCHDOG-REMOTE defs.scp_yamlfile = path.join(defs.remote_jobcatalog_env,'scp_jobfile.'+defs.configuration_identifier) + '_'+defs.run_identifier+ '.' + str(defs.run_id_number) + '.yml' logfile_watchdog = path.join(defs.local_jobcatalog_env, 'watchdog_'+defs.configuration_identifier + '_'+defs.run_identifier+'_'+str(defs.run_id_number)+'.log') cmdfile_watchdog = path.join(defs.local_jobcatalog_env, 'cmd_watchdog_'+defs.configuration_identifier + '_'+defs.run_identifier+'_'+str(defs.run_id_number)) restart_file = path.join(defs.remote_jobcatalog_env,'palm_restart_' + defs.configuration_identifier + '_'+defs.run_identifier+'.'+str(defs.run_id_number)) #SET PORT AND SSH KEY OPTIONS scp_port_option = '-p ' + str(defs.scp_port) if defs.scp_port else '' ssh_key_option = '-k '+ defs.ssh_key if defs.ssh_key else '' #START watchdog SCRIPT FOR MONITORING dfs.scp_yamlfile VIA SHELL COMMAND IN BACKGROUND watchdog_path = path.join(palm.system.cli_dir(),'palm','cmd','watchdog_remote') #palm/cmd/watchdog_remote command = ' '.join(['nohup',watchdog_path,'--pref', defs.scp_yamlfile, '--log', defs.job_protocol_file_remote, '-r', restart_file, '-d', defs.local_jobcatalog_env, '-i', defs.remote_ip, '-u', defs.remote_username, scp_port_option, ssh_key_option, '-f', str(defs.obs_frequency), '>>', logfile_watchdog, '2>&1']) if defs.do_trace: LOG.info(command) subprocess.Popen([command], stdin = None, stdout = None, stderr = None, close_fds = True, shell = True) LOG.info('\n*** subprocess "watchdog_remote" has been started') LOG.info(' Output for monitoring is stored in file:') LOG.info(' '+ logfile_watchdog) LOG.info('\n*** ATTENTION:') LOG.info(' If the local host is shut down, the script must be restarted by the user.') with open(cmdfile_watchdog,'w') as f: f.write(command + ' &') os.chmod(cmdfile_watchdog, stat.S_IXUSR |stat.S_IRUSR ) LOG.info(' The specific call is stored in file: ') LOG.info(' '+ cmdfile_watchdog) elif defs.create_batch_job: shutil.copy(defs.jobfile,EXP('${local_jobcatalog_env}/${configuration_identifier}_${run_id}' )) remember_path = os.getcwd() palm_chdir(defs.local_jobcatalog_env) LOG.info(' \n*** submit the job') command =EXP('${submit_command} ${configuration_identifier}_${run_id}') if defs.do_trace: LOG.info(command) #SUBMIT COMMAND palm_shell(command) os.remove(EXP('${configuration_identifier}_${run_id}')) palm_chdir(remember_path) os.remove(defs.jobfile) else: LOG.info(EXP('*** jobfile created under name \"${jobfile}\" ')) LOG.info(' no batch-job has been sent!') LOG.info('') LOG.info('+++ palm-cli run finished\n') return 0 except KeyboardInterrupt: LOG.info('') LOG.info('+++ palm-cli run killed by "^C"') return 2 except Exception as exception: LOG.error('') LOG.error('+++ palm-cli run crashed') LOG.error('') # CARRY OUT ERROR-COMMANDS GIVEN IN THE CONFIGURATION FILE (EC:) palm_shell("${commands_error}",ignore_error=True) if palm.traceback: LOG.error( exception, exc_info=True ) else: LOG.error( str(exception) ) return 1 finally: # delete temporary folder if defs.delete_temporary_catalog and not defs.create_batch_job and not defs.create_remote_batch_job: if defs.tempdir != None: if path.isdir( defs.tempdir ): if defs.run_identifier in path.basename(defs.tempdir): # for safety only shutil.rmtree(defs.tempdir, ignore_errors=True) # RETURN TO ORIGIN DIRECTORY if not defs.running_in_batch_mode: palm_chdir('${path_current}') def print_summary( defs): def EXP( message ): return defs.expand( message, keep_unknown=False ) def palm_shell( command, capture_output=False, ignore_error=False ): return palm.system.shell( EXP(command), None, capture_output, ignore_error ) def chunkstring(string, length): return (string[0+i:length+i] for i in range(0, len(string), length)) def print_row(*values, count = 1, sformat = '| %-25s%-45s |'): try: if len(values) == 0: for c in range(count): LOG.info( "| %-70s |","") elif len(values) == 1: if sformat != '': LOG.info( sformat, values[0] ) else: LOG.info( values[0] ) elif len(values) == 2: outlist = list(chunkstring(values[1], 45)) LOG.info( sformat, values[0],outlist[0] ) for val in outlist[1:]: LOG.info( sformat, '', val ) except: LOG.error('') LOG.error('+++ failed to print summary') raise Exception calltime = datetime.datetime.now( datetime.timezone.utc ).strftime('%a %b %d %H:%M:%S UTC %Y') print_row('',sformat='') print_row( "#------------------------------------------------------------------------#" ,sformat ='') print_row(calltime, sformat='| %70s |') print_row('palm-cli run','Rev'+EXP(defs.version), sformat='| %-20s%-50s |') print_row('Version:', EXP('${palm_version_string}'), sformat='| %-20s%-50s |') print_row(count=1) print_row("called on:", EXP("${hostname}"), sformat='| %-25s%-45s |') if defs.create_remote_batch_job: print_row('execution on:',EXP('${configuration_identifier} (username: ${remote_username})')) else: if defs.running_on_remote: print_row('config. identifier:',EXP('${configuration_identifier} (execute on IP: ${remote_ip})')) else: print_row('config. identifier:',EXP('${configuration_identifier} (execute on IP: ${local_ip})')) if defs.running_in_batch_mode: print_row('running in','batch job mode') else: if defs.create_batch_job or defs.create_remote_batch_job: print_row('running in','job creation mode') else: print_row('running in','interactive run mode') if defs.running_in_batch_mode or defs.create_batch_job or defs.create_remote_batch_job: if defs.project_account: print_row('project account number:', EXP("${project_account}")) if defs.cores: if defs.run_coupled_model: column = str(defs.cores) + ' (atmosphere: '+str(defs.cores_atmos) + ', ocean: '+str(defs.cores_ocean)+')' else: column = str(defs.cores) print_row('number of cores:', column) if defs.tasks_per_node > 0: print_row('tasks per node:',EXP( '${tasks_per_node} (number of nodes: '+str(defs.nodes)+')')) if defs.cores: if defs.remaining_cores > 0: print_row('', 'one of the nodes only filled with '+str(defs.remaining_cores)+' tasks') if defs.maximum_parallel_io_streams != defs.cores: print_row('max par io streams:', str(defs.maximum_parallel_io_streams)) if defs.use_openmp: print_row('threads_per_task:', str(defs.threads_per_task)) if defs.create_batch_job or defs.create_remote_batch_job or defs.running_in_batch_mode: print_row('memory demand / PE',str(defs.memory) +' MB') print_row('job cpu time (h:m:s)', str(datetime.timedelta(seconds=defs.cputime)) ) print_row(count=1) if defs.source_list != None: if defs.make_options != "": if defs.make_options == '-j$(nproc)': nproc = palm_shell('nproc',capture_output=True ) print_row('make_options', '-j '+ str(nproc)) else: print_row('make options', defs.make_options ) print_row('cpp directives', defs.cpp_options) print_row('compiler options', defs.compiler_options) print_row('linker options:', defs.linker_options) if defs.login_init_cmd: print_row('login init commands:', defs.login_init_cmd) if defs.module_commands: print_row('module commands:', defs.module_commands) print_row(count=1) print_row('run identifier:', defs.run_identifier) print_row('activation string list: ', defs.activation_string_list) if defs.ocean_file_appendix: print_row('suffix \"_O\" is added to local files', sformat='| %-70s |') if defs.source_list: print_row(count=1) print_row('Files to be compiled:',sformat='| %-70s |') source_list = defs.source_list.split() for file in source_list: print_row(' '+file, sformat='| %-70s |') print_row("#------------------------------------------------------------------------#",sformat='') def parse_option( prog=None, argv=None ): # NOTE: this will be much simpler, when the bash case # is not supported anymore if argv is None: argv = sys.argv[1:] parser = build_argparse( prog ) opts, ign = parser.parse_known_args( argv ) opts = dict([ opt for opt in vars(opts).items() ]) if 'traceback' in opts: palm.traceback = True # show traceback on error del opts['traceback'] # delete from options if 'show_summary' not in opts: opts['show_summary'] = False if 'silent' in opts else True # NOTE: using a second (help) parser to seperate palm # arguments and cli arguments # parser = argparse.ArgumentParser( add_help=False ) parser.add_argument( '--pref', default=argparse.SUPPRESS ) for option in ['--bash','--log-append', '--no-summary', '--traceback']: parser.add_argument( option, action='store_true', default=argparse.SUPPRESS ) cli, palm_argv = parser.parse_known_args( argv ) cli_argv = [] if 'bash' in cli: cli_argv.append('--bash') if 'log_append' in cli: cli_argv.append('--log-append') if 'no_summary' in cli: cli_argv.append('--no-summary') if 'traceback' in cli or palm.traceback: cli_argv.append('--traceback') return opts, palm_argv, cli_argv def build_argparse( prog=None ): result = argparse.ArgumentParser( prog=prog, description='''command for running PALM jobs''', add_help=False ) result.add_argument( '-h', '--help', dest='help', action='store_true', default=argparse.SUPPRESS, help='show helps' ) ################################################################# # NOTE: These argmunets are also used in test.py # result.add_argument( '-a', metavar='', dest='activation_string_list', type=palm.config.check_arg('-a','activation_string_list',str), action='store', default=argparse.SUPPRESS, help='For steering the handling of input and output files as defined in the file configuration file ..../palm/model/shrare/config/.palm.iofiles. Argument "d3#" means that the parameter/NAMELIST file for steering PALM shall be provided as input file. This is the minimum setting for option -a, because PALM cannot run without this parameter file.' ) result.add_argument( '-A', dest='project_account', metavar='', type=palm.config.check_arg( '-A', 'project_account', str ), action='store', default=argparse.SUPPRESS, help='project account number' ) result.add_argument( '-b', dest='create_batch_job', action='store_true', default=argparse.SUPPRESS, help='create a batch job' ) result.add_argument( '-B', dest='delete_temporary_catalog', action='store_false', default=argparse.SUPPRESS, help='Do not delete the temporary working directory' ) result.add_argument( '-c', metavar='', dest='configuration_identifier', type=palm.config.check_arg('-c', 'configuration_identifier', str), action='store', default='default', help='Specifies the so-called configuration identifier. It tells palmrun which configuration file should be used. -c default means to use the configuration file .palm.config.default.' ) result.add_argument( '-C', dest='restart_run', action='store_true', default=argparse.SUPPRESS, help='Tells that it is a palmrun call for a restart job that has been automatically created. This is an internal option but it can be used for manually generated restart runs, if the user likes to re-use the contents of the SOURCES_FOR_RUN... folder.' ) result.add_argument( '-F', dest='create_jobfile_only', action='store_true', default=argparse.SUPPRESS, help='Create a batch job file only, and do not submit it.' ) #result.add_argument( # '-G', # metavar='', # dest='palm_version_string', #global_revision (in svn) # type=palm.config.check_arg('-G', 'palm_version_string', str), # action='store', # default=argparse.SUPPRESS, # help='Global revision number of the PALM code' # ) result.add_argument( '-i', dest='run_id_number', type=palm.config.check_arg('-i', 'run_id_number', str), metavar='', action='store', default=argparse.SUPPRESS, help='Five digit random number that gives a run-id and that is used as part of the batch job name as well as the name of the temporary working directory and other files. A new random number is created for each call of palmrun (either a manual call by the user or an automatic call for generating a restart job), and is passed to the batch job internal call of palmrun via this option. ' ) result.add_argument( '-j', dest='running_in_batch_mode', action='store_true', default=argparse.SUPPRESS, help='Tells that palmrun is running within a batch job' ) result.add_argument( '-k', dest='keep_data_from_previous_run', action='store_true', default=argparse.SUPPRESS, help='If set true, input files that have the ln attribute and that have been generated by a previous run within a job chain will be automatically deleted at the end of the run.' ) result.add_argument( '-m', metavar='', dest='memory', type=palm.config.check_arg( '-m', 'memory', int ), action='store', default=argparse.SUPPRESS, help='memory in MByte to be requested in batch jobs per MPI task' ) result.add_argument( '-M', metavar='', dest='makefile', type=palm.config.check_arg('-M', 'makefile', str), action='store', default=argparse.SUPPRESS, help='Makefile to compile the PALM code and utility programs. By default, the name of the makefile is Makefile, and it is expected to be in the folder "{Version}/packages/palm/model/src".' ) result.add_argument( '-O', metavar='', dest='threads_per_task', type=palm.config.check_arg('-O', 'threads_per_task', int), action='store', default=argparse.SUPPRESS, help='OpenMP threads to be started per MPI task. Environment variable OMP_NUM_THREADS will be set to this value' ) result.add_argument( '-q', metavar='', dest='defaultqueue', type=palm.config.check_arg( '-q', 'defaultqueue', str ), action='store', default=argparse.SUPPRESS, help='name of the job queue to which batch jobs will be submitted. See your batch system documentation about available queues and keep in mind that usually each queue has special limits for requested resources.' ) result.add_argument( '-r', metavar='', dest='run_identifier', type=palm.config.check_arg('-r', 'run_identifier', str), action='store', default=argparse.SUPPRESS, help='The name of the run given by -r tells palmrun to use the NAMELIST file _p3d from JOBS//INPUT. It also determines folders and names of output files generated by PALM using informations from the default file configuration file .palm.iofiles. Chapter PALM iofiles explains the format of this file and how you can modify or extend it.' ) result.add_argument( '-R', metavar='', dest='return_address', type=palm.config.check_arg('-R', 'return_address', str), action='store', default=argparse.SUPPRESS, help='Return address. Tells the remote batch job to which IP-address the PALM output and the job protocol file has to be send, and from which machine automatic restarts have to be generated.' ) result.add_argument( '-s', metavar='', dest='source_list', type=palm.config.check_arg('-s', 'source_list', str), action='store', default=argparse.SUPPRESS, help='List of subroutines (Fortran file names) from the GIT repository (under .../palm/model/src) that shall be compiled for this run. Compiled files will be exclusively used for the run and not be put in the MAKE_DEPOSITORY. In case of -s LM, all files in the repository that have been modified by the used will be compiled.' ) result.add_argument( '-t', metavar='', dest='cpumax', type=palm.config.check_arg( '-t', 'cpumax', int ), action='store', default=argparse.SUPPRESS, help='maximum CPU time (wall clock time) in seconds requested by the batch job. This option is ignored in interactive runs.' ) result.add_argument( '-T', metavar='', dest='tasks_per_node', type=palm.config.check_arg( '-T', 'tasks_per_node', int ), action='store', default=argparse.SUPPRESS, help='number of MPI tasks to be started on one node of the computer system. Typically, is chosen as the total number of CPU cores available on one node, e.g. if a node has two CPUs with 12 cores each, then = 24.' ) result.add_argument( '-u', metavar='', dest='remote_username', type=palm.config.check_arg('-u', 'remote_username', str), action='store', default=argparse.SUPPRESS, help='Username on the remote host as given in the configuration file by variable remote_username' ) result.add_argument( '-U', metavar='', dest='local_username', type=palm.config.check_arg('-U', 'local_username', str), action='store', default=argparse.SUPPRESS, help='Username on the local host as given in the configuration file by variable local_username' ) result.add_argument( '-v', dest='silent', action='store_true', default=argparse.SUPPRESS, help='Suppresses parts of palmrun\'s terminal output and prevents palmrun queries' ) result.add_argument( '-V', dest='use_existing_sources_folder', action='store_true', default=argparse.SUPPRESS, help='Use existing SOURCES_FOR_RUN_... folder. Prevents palmrun from creating a new SOURCES_FOR_RUN_... folder. Use this option if you do not want the user interface files to be compiled again.' ) result.add_argument( '-w', metavar='', dest='maximum_parallel_io_streams', type=palm.config.check_arg( '-w', 'maximum_parallel_io_streams', int ), action='store', default=argparse.SUPPRESS, help='as -X Number of parallel I/O streams to be opened by PALM. In the default case, all MPI processes write at the same time. This may cause file system problems in case of a very large number of cores.' ) result.add_argument( '-W', metavar='', dest='previous_job', type=palm.config.check_arg('-W', 'previous_job', str), action='store', default=argparse.SUPPRESS, help='Name (id) of a previous job. Can be used as variable {{previous_job}} as part of job directives in the configuration file, in order to prevent the job to start before the specified previous job has been finished. The job name must be the one that have been given by the batch system.' ) result.add_argument( '-x', dest='do_trace', action='store_true', default=argparse.SUPPRESS, help='Causes palmrun to output excessive debug information for both interactive sessions as well as batch jobs.' ) result.add_argument( '-X', metavar='', dest='cores', type=palm.config.check_arg( '-X', 'cores', int ), action='store', default=argparse.SUPPRESS, help='Total number of cores (not CPUs!) to be used for the run. The argument should not be larger than the maximum number of cores available on your computer (except in case of hyperthreading), because that would usually slow down the performance significantly.' ) result.add_argument( '-y', dest='ocean_file_appendix', action='store_true', default=argparse.SUPPRESS, help='Use file appendix _O for local PALM-I/O files in case of uncoupled ocean runs, e.g. if the run is a precursor run and files shall later be used for coupled atmosphere-ocean runs.' ) result.add_argument( '-Y', metavar='', dest='coupled_dist', type=palm.config.check_arg( '-Y', 'coupled_dist', str), action='store', default=argparse.SUPPRESS, help='In case of a coupled atmosphere-ocean run, the parameter tells PALM how many cores shall be assigned to the atmosphere- and ocean-model, respectively. For example, in case of -X 64 -Y "16 48"" 16 cores are assigned to the atmosphere model, and 48 cores to the ocean model.' ) result.add_argument( '-z', dest='running_in_test_mode', action='store_true', default=argparse.SUPPRESS, help='running in test mode' ) result.add_argument( '-Z', dest='combine_plot_fields', action='store_false', default=argparse.SUPPRESS, help='Do not call combine_plot_fields after PALM has finished. In that case, data output of 2d-cross section or 3d-volumes that has been done be each core into a separate file will not be collected into one file. In order to later process these files, option -B should be set too. -Z might be required for very large jobs in order to reduce computational demands, because combine_plot_fields is running on one core only, so that all other cores will run idle.' ) # ################################################################# result.add_argument( '--bash', dest='use_bash_script', action='store_true', default=False, help='build PALM configuration from YAML configuration and call original bash script \'palmbuild\' or \'palmrun\'' ) result.add_argument( '--log-append', dest='log_append', action='store_true', default=argparse.SUPPRESS, help='Append logging output to existing file otherwise overwrite existing file ' ) result.add_argument( '--no-summary', dest='show_summary', action='store_false', default=argparse.SUPPRESS, help='Do not show parameter summary . If not set the behavior depends on parameter -v' ) result.add_argument( '--traceback', dest='traceback', action='store_true', default=argparse.SUPPRESS, help='Show python\'s traceback on error ' ) return result if __name__ == "__main__": sys.exit( main() )