Skip to content
Snippets Groups Projects
Commit a71f4cfc authored by Sergio Orlandini's avatar Sergio Orlandini
Browse files

update to version v0.4.0 

Merge branch 'develop'

Conflicts:
	submit/config.py
parents da3b4e2f 3f0687cc
Branches master
Tags v0.4.0
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 324 additions and 259 deletions
submit → bin/submit
+ 1
− 1
View file @ a71f4cfc
#!/usr/bin/env python
import sys
from submit import main
from submit.submit import main
if __name__ == "__main__":
sys.exit(main())
......
setup.py
+ 4
− 4
View file @ a71f4cfc
#!/usr/bin/env python
from distutils.core import setup
import config
from submit import config
setup(
......@@ -11,8 +11,8 @@ setup(
author=config.author,
author_email=config.author_email,
url=config.url,
packages=['.', 'argconfig'],
scripts=['submit'],
package_data = {'.': ['config.ini']},
packages=['submit', 'submit/argconfig'],
scripts=['bin/submit'],
package_data = {'submit' : ['config.ini']},
)
submit.py deleted 100755 → 0
+ 0
− 251
View file @ da3b4e2f
#!/usr/bin/env python
import os
import sys
import argparse
import subprocess
from argconfig.argconfig import argconfig as argcfg
argcfg.add_path( os.path.dirname( os.path.realpath(__file__) ) )
# add $HOME/.submit path to config.ini searching paths
argcfg.add_path( os.path.join( os.path.expanduser('~'), '.submit') )
import config
__all__ = ['submit']
_cores = open('/proc/cpuinfo').read().count('processor\t : ')
# default options
default = {
# resources settings
'nodes' : argcfg(dest='nodes').get_default(1),
'cores' : argcfg(dest='cores').get_default(_cores),
'mpis' : argcfg(dest='mpis').get_default(1),
'gpus' : argcfg(dest='gpus').get_default(),
# cluser settings
'queue' : argcfg(dest='queue').get_default(),
'group' : argcfg(dest='group').get_default(),
'account' : argcfg(dest='account').get_default(),
# walltime settings
'hours' : argcfg(dest='hours').get_default(1),
'minutes' : argcfg(dest='minutes').get_default(0),
'seconds' : argcfg(dest='seconds').get_default(0),
}
def exe(cmd, dry_run=False):
print cmd
if dry_run:
return
subprocess.call(cmd, shell=True)
def submit (*args, **kwargs):
"""submit function.
Submit job to pbs scheduler.
"""
d = default
d.update( kwargs )
cmd = ['qsub']
# select resources per node
resources_per_node = ''
# cores
if d.get('cores'):
resources_per_node += ':ncpus=%d' % int(d['cores'])
# mpi processes per node
if d.get('mpis'):
resources_per_node += ':mpiprocs=%d' % int(d['mpis'])
# gpus per node
if d.get('gpus'):
resources_per_node += ':ngpus=%d' % int(d['gpus'])
# select resources
resources = 'select='
for i, node in enumerate(d.get('nodes_list', [])):
if i != 0: resources += '+'
resources += '1'
resources += resources_per_node
resources += ':host=%s' % node
if not d.get('nodes_list'):
resources += "%d" % d['nodes']
resources += resources_per_node
cmd.append('-l ' + resources)
# select timing
_time = [ d.get('hours', 0), d.get('minutes', 0), d.get('seconds', 0) ]
_time = [ int(i) if i else 0 for i in _time]
if [i for i in _time if i]:
_time = 'walltime=%02d:%02d:%02d' % (_time[0], _time[1], _time[2])
cmd.append('-l ' + _time)
# select other options
if d.get('queue', None):
cmd.append('-q' + d['queue'])
if d.get('name', None):
cmd.append('-N ' + d['name'])
if d.get('modules', None):
cmd.append('-V')
if d.get('account', None):
cmd.append('-A' + d['account'])
if d.get('group', None):
cmd.append('-W group_list=' + d['group'])
if d.get('interactive', None) or not d.get('files', None):
# if no files is specified select interactive mode
cmd.append('-I')
if d.get('files', None):
for i in d['files']:
c = list(cmd)
c.append(i)
exe( " ".join(c), d.get('dry_run') )
else:
exe( " ".join(cmd), d.get('dry_run') )
# parse command line options
parser = argparse.ArgumentParser(prog='submit',
description=config.description,
conflict_handler='resolve',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# parse resources
parser.add_argument('-n', '--nodes',
dest='nodes',
default=default['nodes'],
type=int,
help='nodes')
parser.add_argument('-c', '--cores',
dest='cores',
default=default['cores'],
type=int,
help='cores')
parser.add_argument('-m', '--mpis',
dest='mpis',
default=default['mpis'],
type=int,
help='mpis')
parser.add_argument('--no-mpis', dest='mpis', action='store_false')
parser.add_argument('-g', '--gpus',
dest='gpus',
default=default['gpus'],
type=int,
help='gpus')
parser.add_argument('--no-gpus', dest='gpus', action='store_false')
parser.add_argument('--get-nodes',
dest='nodes_list', nargs='*',
default=[],
help='list of nodes to select')
# parse timing
parser.add_argument('-H', '--hours',
dest='hours',
default=default['hours'],
type=int,
help='hours')
parser.add_argument('-M', '--minutes',
dest='minutes',
default=default['minutes'],
type=int,
help='minutes')
parser.add_argument('-S', '--seconds',
dest='seconds',
default=default['seconds'],
type=int,
help='seconds')
# parse other options
parser.add_argument('-q', '--queue',
dest='queue',
default=default['queue'],
help='queue')
parser.add_argument('-A', '--account',
default=default['account'],
help='account')
parser.add_argument('-G', '--group',
default=default['group'],
help='Add group_list option')
parser.add_argument('-N', '--name',
dest='name',
help='name')
parser.add_argument('-v', '--modules',
dest = 'modules',
action="store_true", default = True,
help='export modules')
parser.add_argument('-i', '--interactive',
dest = 'interactive',
action="store_true", default = False,
help='interactive')
parser.add_argument('--dry-run',
dest='dry_run',
action='store_true',
help='dry-run mode')
parser.add_argument('--debug',
dest='debug',
action='store_true',
help='debug mode')
parser.add_argument('--version', action='version',
version='%(prog)s ' + config.version,
help='print version information')
# Positional parameters (Files)
parser.add_argument('files',
default=None, nargs='*',
help = "File1 [File2 ... ]")
def main():
args = parser.parse_args()
if args.debug:
print vars( args )
sys.exit(0)
submit( **vars( args ) )
if __name__ == '__main__':
main()
__init__.py → submit/__init__.py
+ 0
− 0
View file @ a71f4cfc
File moved
argconfig/__init__.py → submit/argconfig/__init__.py
+ 0
− 0
View file @ a71f4cfc
File moved
argconfig/argconfig.py → submit/argconfig/argconfig.py
+ 0
− 0
View file @ a71f4cfc
File moved
config.ini → submit/config.ini
+ 0
− 0
View file @ a71f4cfc
File moved
config.py → submit/config.py
+ 1
− 1
View file @ a71f4cfc
#!/usr/bin/env python
name = 'submit'
version = '0.3.5'
version = '0.4.0'
description = """
Submit job to PBS.
......
submit/submit.py 0 → 100644
+ 301
− 0
View file @ a71f4cfc
#!/usr/bin/env python
import os
import sys
import argparse
import subprocess
import utils
import config
from argconfig.argconfig import argconfig as argcfg
__all__ = ['submit']
argcfg.add_path( os.path.dirname( os.path.realpath(__file__) ) )
# add $HOME/.submit path to config.ini searching paths
argcfg.add_path( os.path.join( os.path.expanduser('~'), '.submit') )
argcfg.set_section( ['default', utils.get_hostname()] )
_cores = open('/proc/cpuinfo').read().count('processor')
# default options
default = {
# resources settings
'nodes' : argcfg(dest='nodes').get_default(1),
'cores' : argcfg(dest='cores').get_default(_cores),
'mpis' : argcfg(dest='mpis').get_default(1),
'gpus' : argcfg(dest='gpus').get_default(),
# cluser settings
'queue' : argcfg(dest='queue').get_default(),
'group' : argcfg(dest='group').get_default(),
'account' : argcfg(dest='account').get_default(),
# walltime settings
'hours' : argcfg(dest='hours').get_default(1),
'minutes' : argcfg(dest='minutes').get_default(0),
'seconds' : argcfg(dest='seconds').get_default(0),
# log default settings
'output' : argcfg(dest='output').get_default(),
'error' : argcfg(dest='error').get_default(),
'joe' : argcfg(dest='joe').get_default(),
'debug' : False,
}
class submit (dict):
def __init__ (self, *args, **kwargs):
self.update(default)
self.update(kwargs)
self['command'] = ['qsub']
# select resources per node
resources_per_node = ''
# cores
if self.get('cores'):
resources_per_node += ':ncpus=%d' % int(self['cores'])
# mpi processes per node
if self.get('mpis'):
resources_per_node += ':mpiprocs=%d' % int(self['mpis'])
# gpus per node
if self.get('gpus'):
resources_per_node += ':ngpus=%d' % int(self['gpus'])
# select resources
resources = 'select='
for i, node in enumerate(self.get('nodes_list', [])):
if i != 0: resources += '+'
resources += '1'
resources += resources_per_node
resources += ':host=%s' % node
if not self.get('nodes_list'):
resources += "%d" % self['nodes']
resources += resources_per_node
self['command'].append('-l ' + resources)
# select timing
_time = [ self.get('hours', 0), self.get('minutes', 0), self.get('seconds', 0) ]
_time = [ int(i) if i else 0 for i in _time]
if [i for i in _time if i]:
_time = 'walltime=%02d:%02d:%02d' % (_time[0], _time[1], _time[2])
self['command'].append('-l ' + _time)
# select other options
if self.get('queue'):
self['command'].append('-q' + self['queue'])
if self.get('name'):
self['command'].append('-N ' + self['name'])
if self.get('modules'):
self['command'].append('-V')
# select account options
if self.get('account'):
self['command'].append('-A' + self['account'])
if self.get('group'):
self['command'].append('-W group_list=' + self['group'])
# select log options
if self.get('output'):
self['command'].append('-o ' + self['output'])
if self.get('error'):
self['command'].append('-e ' + self['error'])
if self.get('joe'):
self['command'].append('-j oe')
if self.get('interactive') or not self.get('files'):
# if no files is specified select interactive mode
self['command'].append('-I')
def submit (self):
if self.get('files'):
for i in self['files']:
c = list(self['command'])
c.append(i)
self._exe( " ".join(c))
else:
self._exe( " ".join(self['command']))
def _exe (self, cmd):
print cmd
if self['debug']:
return
subprocess.call(cmd, shell=True)
# parse command line options
parser = argparse.ArgumentParser(prog='submit',
description=config.description,
conflict_handler='resolve',
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('--version', action='version',
version='%(prog)s ' + config.version,
help='print version information')
parser.add_argument('--dry-run', '--debug',
dest='debug',
action='store_true',
help='dry-run mode')
# parse job options
parser_job = parser.add_argument_group('job options')
parser_job.add_argument('-N', '--name',
dest='name',
help='name')
parser_job.add_argument('-v', '--modules',
dest = 'modules',
action="store_true",
default = True,
help='export modules (default %(default)s)')
parser_job.add_argument('-i', '--interactive',
dest = 'interactive',
action="store_true",
default = False,
help='interactive (default %(default)s)')
# parse resources
parser_resources = parser.add_argument_group('resources options')
parser_resources.add_argument('-n', '--nodes',
dest='nodes',
default=default['nodes'],
type=int,
help='nodes (default %(default)s)')
parser_resources.add_argument('--get-nodes',
dest='nodes_list', nargs='*',
default=[],
help='list of nodes to select')
parser_resources.add_argument('-c', '--cores',
dest='cores',
default=default['cores'],
type=int,
help='cores (default %(default)s)')
parser_resources.add_argument('--no-cores', dest='cores', action='store_false')
parser_resources.add_argument('-m', '--mpis',
dest='mpis',
default=default['mpis'],
type=int,
help='mpis (default %(default)s)')
parser_resources.add_argument('--no-mpis', dest='mpis', action='store_false')
parser_resources.add_argument('-g', '--gpus',
dest='gpus',
default=default['gpus'],
type=int,
help='gpus (default %(default)s)')
parser_resources.add_argument('--no-gpus', dest='gpus', action='store_false')
# parse timing
parser_time = parser.add_argument_group('time options')
parser_time.add_argument('-H', '--hours',
dest='hours',
type=int,
default=default['hours'],
help='hours (default %(default)s)')
parser_time.add_argument('-M', '--minutes',
dest='minutes',
type=int,
default=default['minutes'],
help='minutes (default %(default)s)')
parser_time.add_argument('-S', '--seconds',
dest='seconds',
type=int,
default=default['seconds'],
help='seconds (default %(default)s)')
# queue options
parser_queue = parser.add_argument_group('queue options')
parser_queue.add_argument('-q', '--queue',
dest='queue',
default=default['queue'],
help='queue (default %(default)s)')
# accounting options
parser_account = parser.add_argument_group('accounting options')
parser_account.add_argument('-A', '--account',
default=default['account'],
help='account (default %(default)s)')
parser_account.add_argument('-G', '--group',
default=default['group'],
help='Add group_list option (default %(default)s)')
# log options
parser_log = parser.add_argument_group('log options')
parser_log.add_argument('-o', '--output',
dest='output',
default=default['output'],
help='output (default %(default)s)')
parser_log.add_argument('-e', '--error',
dest='error',
default=default['error'],
help='error (default %(default)s)')
parser_log.add_argument('--joe',
dest='joe',
default=default['joe'],
action='store_true',
help='combine error and output streams (default %(default)s)')
# Positional parameters (Files)
parser.add_argument('files',
default=None, nargs='*',
help = "File1 [File2 ... ]")
def main():
args = parser.parse_args()
a = submit( **vars( args ) )
a.submit()
if __name__ == '__main__':
main()
submit/utils.py 0 → 100644
+ 12
− 0
View file @ a71f4cfc
#!/usr/bin/env python
import socket
hostname_default = 'test'
def get_hostname ():
try :
hostname = socket.gethostbyaddr(socket.gethostname())[0].split(".")[1]
except:
hostname = hostname_default
return hostname
test_submit.py → tests/test_submit.py
+ 5
− 2
View file @ a71f4cfc
......@@ -20,11 +20,14 @@ print submit.default
print "\nSubmit command with default values:"
submit.submit( **submit.default )
a = submit.submit( **submit.default )
a.submit()
print "\nAdd custom value to default values from submit module:"
d = submit.default
d['cores'] = 1
submit.submit( **d )
a = submit.submit( **d )
a.submit()
print
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment