Add Python 3 support

.travis.yml

@@ -1,7 +1,8 @@
 language: python
 python:
  - "2.7"
- #- "3.4"
+ - "3.4"
+ - "3.5"
 
 install:
  - pip install coveralls

scripts/smart-dispatch

@@ -1,5 +1,6 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
+from __future__ import print_function
 
 import os
 import sys
@@ -101,7 +102,7 @@ def main():
 
  # Generating all the worker commands
  worker_script = pjoin(os.path.dirname(smartdispatch.__file__), 'workers', 'base_worker.py')
- COMMAND_STRING = 'cd "{cwd}"; python2 {worker_script} "{commands_file}" "{log_folder}" '\
+ COMMAND_STRING = 'cd "{cwd}"; python {worker_script} "{commands_file}" "{log_folder}" '\
  '1>> "{log_folder}/worker/$PBS_JOBID\"\"_worker_{{ID}}.o" '\
  '2>> "{log_folder}/worker/$PBS_JOBID\"\"_worker_{{ID}}.e" '
  COMMAND_STRING = COMMAND_STRING.format(cwd=os.getcwd(), worker_script=worker_script, commands_file=command_manager._commands_filename, log_folder=path_job_logs)
@@ -119,19 +120,23 @@ def main():
  pbs_filenames = job_generator.write_pbs_files(path_job_commands)
 
  # Launch the jobs
- print"## {nb_commands} command(s) will be executed in {nb_jobs} job(s) ##".format(nb_commands=nb_commands, nb_jobs=len(pbs_filenames))
- print"Batch UID:\n{batch_uid}".format(batch_uid=jobname)
+ print("## {nb_commands} command(s) will be executed in {nb_jobs} job(s) ##".format(nb_commands=nb_commands, nb_jobs=len(pbs_filenames)))
+ print("Batch UID:\n{batch_uid}".format(batch_uid=jobname))
  if not args.doNotLaunch:
  jobs_id = []
  for pbs_filename in pbs_filenames:
  qsub_output = check_output('{launcher} {pbs_filename}'.format(launcher=LAUNCHER if args.launcher is None else args.launcher, pbs_filename=pbs_filename), shell=True)
+
+ if isinstance(qsub_output, bytes):
+ qsub_output = qsub_output.decode("utf-8")
+
  jobs_id += [qsub_output.strip()]
 
  with open_with_lock(pjoin(path_job, "jobs_id.txt"), 'a') as jobs_id_file:
  jobs_id_file.writelines(t.strftime("## %Y-%m-%d %H:%M:%S ##\n"))
  jobs_id_file.writelines("\n".join(jobs_id) + "\n")
- print"\nJobs id:\n{jobs_id}".format(jobs_id=" ".join(jobs_id))
- print"\nLogs, command, and jobs id related to this batch will be in:\n {smartdispatch_folder}".format(smartdispatch_folder=path_job)
+ print("\nJobs id:\n{jobs_id}".format(jobs_id=" ".join(jobs_id)))
+ print("\nLogs, command, and jobs id related to this batch will be in:\n {smartdispatch_folder}".format(smartdispatch_folder=path_job))
 
 
 def parse_arguments():
@@ -146,7 +151,8 @@ def parse_arguments():
  parser.add_argument('-c', '--coresPerCommand', type=int, required=False, help='How many cores a command needs.', default=1)
  parser.add_argument('-g', '--gpusPerCommand', type=int, required=False, help='How many gpus a command needs.', default=1)
  # parser.add_argument('-m', '--memPerCommand', type=float, required=False, help='How much memory a command needs (in Gb).')
- parser.add_argument('-f', '--commandsFile', type=file, required=False, help='File containing commands to launch. Each command must be on a seperate line. (Replaces commandAndOptions)')
+ parser.add_argument('-f', '--commandsFile', type=argparse.FileType('r'), required=False,
+ help='File containing commands to launch. Each command must be on a seperate line. (Replaces commandAndOptions)')
 
  parser.add_argument('-l', '--modules', type=str, required=False, help='List of additional modules to load.', nargs='+')
  parser.add_argument('-x', '--doNotLaunch', action='store_true', help='Creates the QSUB files without launching them.')

smartdispatch/argument_template.py

@@ -35,7 +35,7 @@ def unfold(self, match):
  start = int(groups[0])
  end = int(groups[1])
  step = 1 if groups[2] is None else int(groups[2])
- return map(str, range(start, end, step))
+ return [str(i) for i in range(start, end, step)]
 
 
 argument_templates = build_argument_templates_dictionnary()

smartdispatch/smartdispatch.py

@@ -97,7 +97,7 @@ def unfold_command(command):
  pos = match.end()
 
  arguments.append([text[pos:]]) # Add remaining unfolded arguments
- arguments = [map(utils.decode_escaped_characters, argvalues) for argvalues in arguments]
+ arguments = [[utils.decode_escaped_characters(v) for v in argvalues] for argvalues in arguments]
  return ["".join(argvalues) for argvalues in itertools.product(*arguments)]
 
 

smartdispatch/tests/test_filelock.py

@@ -38,9 +38,9 @@ def _test_open_with_lock(lock_func):
  time.sleep(1)
 
  stdout, stderr = process.communicate()
- assert_equal(stdout, "")
- assert_true("Traceback" not in stderr, msg="Unexpected error: '{}'".format(stderr))
- assert_true("write-lock" in stderr, msg="Forcing a race condition, try increasing sleeping time above.")
+ assert_equal(stdout, b"")
+ assert_true("Traceback" not in stderr.decode(), msg="Unexpected error: '{}'".format(stderr.decode()))
+ assert_true("write-lock" in stderr.decode(), msg="Forcing a race condition, try increasing sleeping time above.")
 
  shutil.rmtree(temp_dir) # Cleaning up.
 

smartdispatch/tests/test_smartdispatch.py

@@ -3,7 +3,7 @@
 import shutil
 import time as t
 from os.path import join as pjoin
-from StringIO import StringIO
+from io import StringIO, open
 
 import tempfile
 from nose.tools import assert_true, assert_equal
@@ -43,11 +43,11 @@ def test_get_commands_from_file():
  commands = ["command1 arg1 arg2",
  "command2",
  "command3 arg1 arg2 arg3 arg4"]
- fileobj = StringIO("\n".join(commands))
+ fileobj = StringIO(u"\n".join(commands))
  assert_array_equal(smartdispatch.get_commands_from_file(fileobj), commands)
 
  # Test stripping last line if empty
- fileobj = StringIO("\n".join(commands) + "\n")
+ fileobj = StringIO(u"\n".join(commands) + u"\n")
  assert_array_equal(smartdispatch.get_commands_from_file(fileobj), commands)
 
 

smartdispatch/utils.py

@@ -1,4 +1,7 @@
+from __future__ import print_function
+
 import re
+import binascii
 import hashlib
 import unicodedata
 import json
@@ -15,7 +18,7 @@ def print_boxed(string):
  out = u"\u250c" + box_line + u"\u2510\n"
  out += '\n'.join([u"\u2502 {} \u2502".format(line.ljust(max_len)) for line in splitted_string])
  out += u"\n\u2514" + box_line + u"\u2518"
- printout
+ print(out)
 
 
 def yes_no_prompt(query, default=None):
@@ -35,13 +38,13 @@ def yes_no_prompt(query, default=None):
 
 def chunks(sequence, n):
  """ Yield successive n-sized chunks from sequence. """
- for i in xrange(0, len(sequence), n):
+ for i in range(0, len(sequence), n):
  yield sequence[i:i + n]
 
 
 def generate_uid_from_string(value):
  """ Create unique identifier from a string. """
- return hashlib.sha256(value).hexdigest()
+ return hashlib.sha256(value.encode()).hexdigest()
 
 
 def slugify(value):
@@ -54,15 +57,23 @@ def slugify(value):
  ---------
  https://github.com/django/django/blob/1.7c3/django/utils/text.py#L436
  """
- value = unicodedata.normalize('NFKD', unicode(value, "UTF-8")).encode('ascii', 'ignore').decode('ascii')
+ # Convert `value` to Unicode so we can slugify it using the unicodedata module.
+ try:
+ value = unicode(value, "UTF-8")
+ except NameError:
+ pass # In Python 3, all strings are already stored as Unicode.
+
+ # Replace all compatibility characters with their equivalents.
+ value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii')
  value = re.sub('[^\w\s-]', '', value).strip().lower()
  return str(re.sub('[-\s]+', '_', value))
 
 
 def encode_escaped_characters(text, escaping_character="\\"):
  """ Escape the escaped character using its hex representation """
  def hexify(match):
- return "\\x{0}".format(match.group()[-1].encode("hex"))
+ # Reference: http://stackoverflow.com/questions/18298251/python-hex-values-to-convert-to-a-string-integer
+ return "\\x" + binascii.hexlify(match.group()[-1].encode()).decode()
 
  return re.sub(r"\\.", hexify, text)
 
@@ -73,7 +84,7 @@ def decode_escaped_characters(text):
  return ''
 
  def unhexify(match):
- return match.group()[2:].decode("hex")
+ return binascii.unhexlify(match.group()[2:]).decode()
 
  return re.sub(r"\\x..", unhexify, text)
 
@@ -92,6 +103,8 @@ def detect_cluster():
  # Get server status
  try:
  output = Popen(["qstat", "-B"], stdout=PIPE).communicate()[0]
+ if isinstance(output, bytes):
+ output = output.decode("utf-8")
  except OSError:
  # If qstat is not available we assume that the cluster is unknown.
  return None

smartdispatch/workers/base_worker.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 import os

smartdispatch/workers/tests/test_base_worker.py

@@ -25,14 +25,14 @@ def setUp(self):
  self.command_manager = CommandManager(os.path.join(self._commands_dir, "commands.txt"))
  self.command_manager.set_commands_to_run(self.commands)
 
- self.commands_uid = map(utils.generate_uid_from_string, self.commands)
+ self.commands_uid = [utils.generate_uid_from_string(c) for c in self.commands]
 
  def tearDown(self):
  shutil.rmtree(self._commands_dir)
  shutil.rmtree(self.logs_dir)
 
  def test_main(self):
- command = ['python2', self.base_worker_script, self.command_manager._commands_filename, self.logs_dir]
+ command = ['python', self.base_worker_script, self.command_manager._commands_filename, self.logs_dir]
  assert_equal(call(command), 0)
  # Simulate a resume, i.e. re-run the command, the output/error should be concatenated.
  self.command_manager.set_commands_to_run(self.commands)
@@ -106,14 +106,14 @@ def test_main(self):
  assert_equal("", logfile.read())
 
  def test_lock(self):
- command = ['python2', self.base_worker_script, self.command_manager._commands_filename, self.logs_dir]
+ command = ['python', self.base_worker_script, self.command_manager._commands_filename, self.logs_dir]
 
  # Lock the commands file before running 'base_worker.py'
  with open_with_lock(self.command_manager._commands_filename, 'r+'):
  process = Popen(command, stdout=PIPE, stderr=PIPE)
  time.sleep(1)
 
  stdout, stderr = process.communicate()
- assert_equal(stdout, "")
- assert_true("write-lock" in stderr, msg="Forcing a race condition, try increasing sleeping time above.")
- assert_true("Traceback" not in stderr) # Check that there are no errors.
+ assert_equal(stdout, b"")
+ assert_true("write-lock" in stderr.decode(), msg="Forcing a race condition, try increasing sleeping time above.")
+ assert_true("Traceback" not in stderr.decode()) # Check that there are no errors.