from collections import namedtuple import os import shlex import signal from string import Template import subprocess import palm LOG = palm.logger(__name__) ######################################################################### # FIXME: PALM (with palm-cli as package) assumes the following directory # sturcture: # # on lcocal host: # # | = # /bin # + palmrun -> /packages/model/bin/palmrun # + palm-cli -> /packages/palm-cli/palm-cli (python) # /rrtmg # .palm.config.default # .palm.config.default.yaml # # /packages # /palm # /cli # / # /palm # + system.py <<< we are here # + palm-cli # /model <<< project_root_dir = root_dir # /bin <<< project_bin_dir = bin_dir # + palmrun # + palmbuild # /build <<< project_build_dir = build_dir # /share # /config # + .palm.iofiles <<< default_palm_iofiles = default_iofiles # /rrtmg # + VERSION # # on remote host: # # / # /MAKE_DEPOSITORY_... # /bin # /palm # + system.py <<< we are here # + palm-cli # + Makefile # + *.f90 # + # + palm-cli -> //MAKE_DEPOSITORY_.../bin/palm-cli # # tmp/|SOURCES_FOR_RUN_... # + Makefile # + *.f90 # + # + palm-cli -> //MAKE_DEPOSITORY_.../bin/palm-cli # # /rrtmg # + VERSION # # # NOTE: (1) it is recommanded to use as workspace # (current working directory) # (2) palmrun/palmbuild expect .palm.config. # in working directory # ######################################################################### def root_dir(): # NOTE: possible strategies for palm-cli to find palm root directory # (1) from called palm-cli: realpath(palm-cli)/../../model # (2) from workspace: realpath(bin/palmrun)/.. # (3) from $PATH: realpaht(whereis palmrun)/.. # # NOTE: root_dir() is None on remote system, so cli_dir(), bin_dir() # build_dir(), script_dir() and default_iofiles() throw an # exception # result = root_dir_from_cli() if result is None: result = root_dir_from_local_palmrun() if result is None: result = root_dir_from_global_palmrun() return result def root_dir_from_cli(): result = None candidate = os.path.join( cli_dir(), '..', '..', 'model' ) if os.path.isdir(candidate): result = os.path.realpath(candidate) return result def root_dir_from_local_palmrun(): result = None candidate = os.path.join( '.', 'bin', 'palmrun' ) if os.path.isfile(candidate): candidate = os.path.realpath(candidate) result = os.path.join( os.path.dirname(candidate), '..' ) result = os.path.normpath(result) return result def root_dir_from_global_palmrun(): result = None candidates = subprocess.check_output(['whereis','palmrun'],universal_newlines=True) candidates = candidates.rstrip().split(' ')[1:] for candidate in candidates: if os.path.isfile(candidate): candidate = os.path.realpath(candidate) result = os.path.join( os.path.dirname(candidate), '..' ) result = os.path.normpath(result) break return result def cli_dir(): result = os.path.join( os.path.dirname(__file__), '..' ) result = os.path.realpath(result) return result def bin_dir(): result = os.path.join( root_dir(), 'bin' ) result = os.path.realpath(result) return result def build_dir(): result = os.path.join( root_dir(), 'build' ) result = os.path.realpath(result) return result def source_dir(): result = os.path.join( root_dir(), 'src' ) result = os.path.realpath(result) return result def scripts_dir(): result = os.path.join( root_dir(), 'bin' ) result = os.path.realpath(result) return result def default_iofiles(): result = os.path.join( root_dir(), 'share', 'config', '.palm.iofiles' ) result = os.path.realpath(result) return result def palm_config_file( identifier=None ): if identifier is None: identifier = 'default' result = '.palm.config.' + str(identifier).strip() result = os.path.join( os.getcwd() , result ) return result def yaml_config_file( identifier=None ): result = palm_config_file(identifier) + '.yml' return result def version(): # NOTE: possible strategies for palm-cli to retrieve version # (1) git # (2) $root_dir/../../VERSION # (2) cwd()/VERSION # result = version_from_git() if result is None: result = version_from_file_in_root() if result is None: result = version_from_file_in_cwd() if result is None: result = '' return result def version_from_git(): try: result = shell( '(cd '+source_dir()+'; git rev-parse --short HEAD) 2>/dev/null', capture_output=True ) result = 'PALM (git SHA-1): '+result return result except: return None def version_from_file_in_root(): try: return version_from_file( os.path.join( root_dir(), '..', '..', '..', 'VERSION' )) except: return None def version_from_file_in_cwd(): try: return version_from_file( os.path.join( os.getcwd(), 'VERSION' )) except: return None def version_from_file(filename): try: version_file = os.path.normpath(filename) with open(version_file) as file: result = file.readline().strip() result = 'PALM '+result return result except: return None def chdir( path, env=None ): if env: path = Template(path).safe_substitute(env) LOG.debug( 'chdir: %s', path ) os.chdir( path ) def run( palmscript, args ): scriptfile = os.path.join( bin_dir(), palmscript ) scriptfile = os.path.realpath(scriptfile) # TODO: adjust routine 'call' like 'shell' to # show std:out und std::error in log cwd = os.getcwd() try: return call( [ scriptfile ] + args ).returncode except KeyboardInterrupt: return 2 finally: os.chdir(cwd) def shell( command, env=None, capture_output=False, ignore_error=False ): if env: command = Template(command).safe_substitute(env) for line in command.splitlines(): LOG.debug( 'shell: %s', line ) # LOG.debug( 'from: %s', os.getcwd() ) # NOTE: we need to use 'bash' ... # shellcmd = "bash -c '{}'".format(command) shellcmd = shlex.split(shellcmd) # NOTE: this may block for capture_output = True if stdout buffers is # filling up # process = subprocess.Popen( shellcmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE if capture_output else subprocess.STDOUT, universal_newlines = True, # set group ID and session ID to process.PID preexec_fn = os.setsid ) try: stream = process.stderr if capture_output else process.stdout for nextline in stream: LOG.info( nextline.rstrip() ) except KeyboardInterrupt: # kill process group # os.killpg( os.getpgid( process.pid ), signal.SIGTERM ) raise result, stderr = process.communicate() if process.returncode != 0 and not ignore_error: raise subprocess.CalledProcessError( process.returncode, command ) return result.strip() CallResult = namedtuple( 'CallResult', ['returncode','stdout','stderr'] ) def call( args, timeout=None, cache_stdout=False, cache_stderr=False, shell=False, verbose=True ): # TODO: adjust routine 'call' like 'shell' to # show std:out und std::error in log # REQUIRE: args is String|list # args = shlex.split(args) if isinstance( args, str ) else args name = os.path.basename( args[0] ) try: if verbose: LOG.debug( 'current dir.: %s', os.getcwd() ) LOG.debug( 'start running: %s', name ) process = subprocess.Popen( args, stdout = subprocess.PIPE if cache_stdout else None, stderr = subprocess.PIPE if cache_stderr else None, shell = shell, universal_newlines = True, # set group ID and session ID to process.PID preexec_fn = os.setsid ) try: stdout, stderr = process.communicate( timeout=timeout ) except KeyboardInterrupt: # kill process group # os.killpg( os.getpgid( process.pid ), signal.SIGTERM ) raise except subprocess.TimeoutExpired: if verbose: LOG.warning( 'timeout expired ...' ) # kill process group # os.killpg( os.getpgid( process.pid ), signal.SIGTERM ) if verbose: LOG.debug( 'stop running: %s', name ) stdout, stderr = process.communicate() return CallResult( process.returncode, stdout, stderr ) except: if verbose: LOG.error( 'fail running: %s', name ) raise