Updates to make smart-dispatch Python 3 compatible

.travis.yml

@@ -1,7 +1,7 @@
 language: python
 python:
  - "2.7"
-#- "3.4"
+ - "3.5"
 
 install:
  - pip install coveralls
@@ -12,4 +12,4 @@ script:
  - nosetests -v --with-coverage . smartdispatch/workers
 
 after_success:
- - coveralls
+ - coveralls

scripts/smart-dispatch

@@ -1,5 +1,6 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
+from __future__ import print_function
 
 import os
 import sys
@@ -178,11 +179,11 @@ def main():
  pbs_filenames = job_generator.write_pbs_files(path_job_commands)
 
  # Launch the jobs
- print"## {nb_commands} command(s) will be executed in {nb_jobs} job(s) ##".format(nb_commands=nb_commands, nb_jobs=len(pbs_filenames))
- print"Batch UID:\n{batch_uid}".format(batch_uid=jobname)
+ print("## {nb_commands} command(s) will be executed in {nb_jobs} job(s) ##".format(nb_commands=nb_commands, nb_jobs=len(pbs_filenames)))
+ print("Batch UID:\n{batch_uid}".format(batch_uid=jobname))
  if not args.doNotLaunch:
  launch_jobs(LAUNCHER if args.launcher is None else args.launcher, pbs_filenames, CLUSTER_NAME, path_job)
- print"\nLogs, command, and jobs id related to this batch will be in:\n {smartdispatch_folder}".format(smartdispatch_folder=path_job)
+ print("\nLogs, command, and jobs id related to this batch will be in:\n {smartdispatch_folder}".format(smartdispatch_folder=path_job))
 
 
 def parse_arguments():
@@ -198,7 +199,7 @@ def parse_arguments():
  parser.add_argument('-c', '--coresPerCommand', type=int, required=False, help='How many cores a command needs.', default=1)
  parser.add_argument('-g', '--gpusPerCommand', type=int, required=False, help='How many gpus a command needs.', default=1)
  # parser.add_argument('-m', '--memPerCommand', type=float, required=False, help='How much memory a command needs (in Gb).')
- parser.add_argument('-f', '--commandsFile', type=file, required=False, help='File containing commands to launch. Each command must be on a seperate line. (Replaces commandAndOptions)')
+ parser.add_argument('-f', '--commandsFile', type=open, required=False, help='File containing commands to launch. Each command must be on a seperate line. (Replaces commandAndOptions)')
 
  parser.add_argument('-l', '--modules', type=str, required=False, help='List of additional modules to load.', nargs='+')
  parser.add_argument('-x', '--doNotLaunch', action='store_true', help='Generate all the files without launching the job.')

setup.py

@@ -14,6 +14,6 @@
  license='LICENSE.txt',
  description='An easy to use job launcher for supercomputers with PBS compatible job manager.',
  long_description=open('README.md').read(),
- install_requires=['psutil>=1'],
+ install_requires=['psutil>=1', 'future'],
  package_data={'smartdispatch': ['config/*.json']}
 )

smartdispatch/argument_template.py

@@ -1,3 +1,6 @@
+from builtins import map
+from builtins import range
+from builtins import object
 import re
 from collections import OrderedDict
 
@@ -35,7 +38,7 @@ def unfold(self, match):
  start = int(groups[0])
  end = int(groups[1])
  step = 1 if groups[2] is None else int(groups[2])
- return map(str, range(start, end, step))
+ return list(map(str, list(range(start, end, step))))
 
 
 argument_templates = build_argument_templates_dictionnary()

smartdispatch/command_manager.py

@@ -1,3 +1,4 @@
+from builtins import object
 import os
 from .filelock import open_with_lock
 

smartdispatch/job_generator.py

@@ -1,5 +1,7 @@
 from __future__ import absolute_import
 
+from builtins import str
+from builtins import object
 import os
 import re
 from smartdispatch.pbs import PBS

smartdispatch/pbs.py

@@ -1,3 +1,5 @@
+from builtins import str
+from builtins import object
 import re
 from collections import OrderedDict
 
@@ -54,7 +56,7 @@ def add_options(self, **options):
  Declares a name for the job. It must consist of printable,
  non white space characters with the first character alphabetic.
  """
- for option_name, option_value in options.items():
+ for option_name, option_value in list(options.items()):
  # If known option, validate it.
  if option_name.strip('-') == 'N':
  if len(option_name) > 64:
@@ -81,7 +83,7 @@ def add_resources(self, **resources):
  *pmem*: pmem=[0-9]+(b|kb|mb|gb|tb)
  Specifies the maximum amount of physical memory used by any single process of the job.
  """
- for resource_name, resource_value in resources.items():
+ for resource_name, resource_value in list(resources.items()):
  # If known ressource, validate it.
  if resource_name == 'nodes':
  if re.match(regex_resource_nodes, str(resource_value)) is None:
@@ -150,13 +152,13 @@ def __str__(self):
  pbs = []
  pbs += ["#!/bin/bash"]
 
- for option_name, option_value in self.options.items():
+ for option_name, option_value in list(self.options.items()):
  if option_value == "":
  pbs += ["#PBS {0}".format(option_name)]
  else:
  pbs += ["#PBS {0} {1}".format(option_name, option_value)]
 
- for resource_name, resource_value in self.resources.items():
+ for resource_name, resource_value in list(self.resources.items()):
  pbs += ["#PBS -l {0}={1}".format(resource_name, resource_value)]
 
  pbs += ["\n# Modules #"]

smartdispatch/queue.py

@@ -1,4 +1,6 @@
-from smartdispatch import get_available_queues
+from __future__ import absolute_import
+from builtins import object
+from .smartdispatch import get_available_queues
 
 
 class Queue(object):

smartdispatch/smartdispatch.py

@@ -1,5 +1,7 @@
-from __future__ import absolute_import
+from __future__ import absolute_import, print_function
 
+from builtins import next
+from builtins import map
 import os
 import re
 import itertools
@@ -89,7 +91,7 @@ def unfold_command(command):
  text = utils.encode_escaped_characters(command)
 
  # Build the master regex with all argument's regex
- regex = "(" + "|".join(["(?P<{0}>{1})".format(name, arg.regex) for name, arg in argument_templates.items()]) + ")"
+ regex = "(" + "|".join(["(?P<{0}>{1})".format(name, arg.regex) for name, arg in list(argument_templates.items())]) + ")"
 
  pos = 0
  arguments = []
@@ -98,12 +100,12 @@ def unfold_command(command):
  arguments.append([text[pos:match.start()]])
 
  # Unfold argument
- argument_template_name, matched_text = next((k, v) for k, v in match.groupdict().items() if v is not None)
+ argument_template_name, matched_text = next((k, v) for k, v in list(match.groupdict().items()) if v is not None)
  arguments.append(argument_templates[argument_template_name].unfold(matched_text))
  pos = match.end()
 
  arguments.append([text[pos:]]) # Add remaining unfolded arguments
- arguments = [map(utils.decode_escaped_characters, argvalues) for argvalues in arguments]
+ arguments = [list(map(utils.decode_escaped_characters, argvalues)) for argvalues in arguments]
  return ["".join(argvalues) for argvalues in itertools.product(*arguments)]
 
 
@@ -185,11 +187,11 @@ def launch_jobs(launcher, pbs_filenames, cluster_name, path_job): # pragma: no
  for pbs_filename in pbs_filenames:
  launcher_output = check_output('PBS_FILENAME={pbs_filename} {launcher} {pbs_filename}'.format(
  launcher=launcher, pbs_filename=pbs_filename), shell=True)
- jobs_id += [launcher_output.strip()]
+ jobs_id += [launcher_output.strip().decode()]
 
  # On some clusters, SRMJID and PBS_JOBID don't match
  if cluster_name in ['helios']:
- launcher_output = check_output(['qstat', '-f']).split('Job Id: ')
+ launcher_output = check_output(['qstat', '-f']).decode().split('Job Id: ')
  for job in launcher_output:
  if re.search(r"SRMJID:{job_id}".format(job_id=jobs_id[-1]), job):
  pbs_job_id = re.match(r"[0-9a-zA-Z.-]*", job).group()
@@ -198,4 +200,4 @@ def launch_jobs(launcher, pbs_filenames, cluster_name, path_job): # pragma: no
  with open_with_lock(pjoin(path_job, "jobs_id.txt"), 'a') as jobs_id_file:
  jobs_id_file.writelines(t.strftime("## %Y-%m-%d %H:%M:%S ##\n"))
  jobs_id_file.writelines("\n".join(jobs_id) + "\n")
- print"\nJobs id:\n{jobs_id}".format(jobs_id=" ".join(jobs_id))
+ print("\nJobs id:\n{jobs_id}".format(jobs_id=" ".join(jobs_id)))

smartdispatch/tests/test_filelock.py

@@ -38,7 +38,7 @@ def _test_open_with_lock(lock_func):
  time.sleep(1)
 
  stdout, stderr = process.communicate()
- assert_equal(stdout, "")
+ assert_equal(stdout, b"")
  assert_true("Traceback" not in stderr, msg="Unexpected error: '{}'".format(stderr))
  assert_true("write-lock" in stderr, msg="Forcing a race condition, try increasing sleeping time above.")
 

smartdispatch/tests/test_job_generator.py

@@ -1,3 +1,5 @@
+from builtins import str
+from builtins import object
 from nose.tools import assert_true, assert_false, assert_equal, assert_raises
 
 import os

smartdispatch/tests/test_pbs.py

@@ -1,3 +1,4 @@
+from builtins import str
 from nose.tools import assert_true, assert_equal, assert_raises
 from numpy.testing import assert_array_equal
 
@@ -29,7 +30,7 @@ def test_constructor(self):
  def test_add_options(self):
  # Default options
  assert_equal(len(self.pbs.options), 2)
- assert_true('-V' in self.pbs.options.keys())
+ assert_true('-V' in list(self.pbs.options.keys()))
  assert_equal(self.pbs.options['-q'], self.queue_name)
 
  self.pbs.add_options(A="option1")

smartdispatch/tests/test_queue.py

@@ -35,7 +35,7 @@ def test_constructor(self):
 
  # Test with missing information but referring to a known queue.
  for cluster_name in self.known_clusters:
- for queue_name, queue_infos in get_available_queues(cluster_name).items():
+ for queue_name, queue_infos in list(get_available_queues(cluster_name).items()):
  queue = Queue(queue_name, cluster_name)
  assert_equal(queue.name, queue_name)
  assert_equal(queue.cluster_name, cluster_name)

smartdispatch/tests/test_smartdispatch.py

@@ -1,9 +1,11 @@
+from future import standard_library
+standard_library.install_aliases()
 import os
 import re
 import shutil
 import time as t
 from os.path import join as pjoin
-from StringIO import StringIO
+from io import StringIO
 
 import tempfile
 from nose.tools import assert_true, assert_equal

smartdispatch/tests/test_utils.py

@@ -1,4 +1,6 @@
 # -*- coding: utf-8 -*-
+from builtins import zip
+from builtins import range
 import unittest
 
 from smartdispatch import utils
@@ -21,11 +23,11 @@ def test_print_boxed_empty(self):
 
 
 def test_chunks():
- sequence = range(10)
+ sequence = list(range(10))
 
  for n in range(1, 11):
  expected = []
- for start, end in zip(range(0, len(sequence), n), range(n, len(sequence) + n, n)):
+ for start, end in zip(list(range(0, len(sequence), n)), list(range(n, len(sequence) + n, n))):
  expected.append(sequence[start:end])
 
  assert_array_equal(list(utils.chunks(sequence, n)), expected, "n:{0}".format(n))

smartdispatch/utils.py

@@ -1,7 +1,21 @@
+from __future__ import print_function
+
+from builtins import str
+from builtins import input
+from builtins import map
+from builtins import range
+
+try:
+ _unicode = unicode
+ _utf8 = lambda x: _unicode(x, 'UTF-8')
+except NameError:
+ _utf8 = str
+
 import re
 import hashlib
 import unicodedata
 import json
+import codecs
 
 from distutils.util import strtobool
 from subprocess import Popen, PIPE
@@ -17,7 +31,7 @@ def jobname_generator(jobname, job_id):
  Returns
  -------
  str
- The cropped version of the string.
+ The cropped version of the string.
  '''
  # 64 - 1 since the total length including -1 should be less than 64
  job_id = str(job_id)
@@ -30,13 +44,13 @@ def jobname_generator(jobname, job_id):
 
 def print_boxed(string):
  splitted_string = string.split('\n')
- max_len = max(map(len, splitted_string))
+ max_len = max(list(map(len, splitted_string)))
  box_line = u"\u2500" * (max_len + 2)
 
  out = u"\u250c" + box_line + u"\u2510\n"
  out += '\n'.join([u"\u2502 {} \u2502".format(line.ljust(max_len)) for line in splitted_string])
  out += u"\n\u2514" + box_line + u"\u2518"
- printout
+ print(out)
 
 
 def yes_no_prompt(query, default=None):
@@ -47,7 +61,7 @@ def yes_no_prompt(query, default=None):
 
  while True:
  try:
- answer = raw_input("{0}{1}".format(query, available_prompts[default]))
+ answer = eval(input("{0}{1}".format(query, available_prompts[default])))
  return strtobool(answer)
  except ValueError:
  if answer == '' and default is not None:
@@ -56,13 +70,13 @@ def yes_no_prompt(query, default=None):
 
 def chunks(sequence, n):
  """ Yield successive n-sized chunks from sequence. """
- for i in xrange(0, len(sequence), n):
+ for i in range(0, len(sequence), n):
  yield sequence[i:i + n]
 
 
 def generate_uid_from_string(value):
  """ Create unique identifier from a string. """
- return hashlib.sha256(value).hexdigest()
+ return hashlib.sha256(value.encode('UTF-8')).hexdigest()
 
 
 def slugify(value):
@@ -75,15 +89,15 @@ def slugify(value):
  ---------
  https://github.com/django/django/blob/1.7c3/django/utils/text.py#L436
  """
- value = unicodedata.normalize('NFKD', unicode(value, "UTF-8")).encode('ascii', 'ignore').decode('ascii')
+ value = unicodedata.normalize('NFKD', _utf8(value)).encode('ascii', 'ignore').decode('ascii')
  value = re.sub('[^\w\s-]', '', value).strip().lower()
  return str(re.sub('[-\s]+', '_', value))
 
 
 def encode_escaped_characters(text, escaping_character="\\"):
  """ Escape the escaped character using its hex representation """
  def hexify(match):
- return "\\x{0}".format(match.group()[-1].encode("hex"))
+ return codecs.encode("\\x{0}".format(match.group()[-1], 'hex_codec'))
 
  return re.sub(r"\\.", hexify, text)
 
@@ -117,7 +131,7 @@ def detect_cluster():
  # If qstat is not available we assume that the cluster is unknown.
  return None
  # Get server name from status
- server_name = output.split('\n')[2].split(' ')[0]
+ server_name = output.decode().split('\n')[2].split(' ')[0]
  # Cleanup the name and return it
  cluster_name = None
  if server_name.split('.')[-1] == 'm':

smartdispatch/workers/tests/test_base_worker.py

@@ -114,6 +114,6 @@ def test_lock(self):
  time.sleep(1)
 
  stdout, stderr = process.communicate()
- assert_equal(stdout, "")
+ assert_equal(stdout, b"")
  assert_true("write-lock" in stderr, msg="Forcing a race condition, try increasing sleeping time above.")
  assert_true("Traceback" not in stderr) # Check that there are no errors.