diff --git a/luigi/contrib/sge.py b/luigi/contrib/sge.py index 51dacd9003..d22d2f402e 100755 --- a/luigi/contrib/sge.py +++ b/luigi/contrib/sge.py @@ -141,8 +141,22 @@ def _parse_qsub_job_id(qsub_out): def _build_qsub_command(cmd, job_name, outfile, errfile, pe, n_cpu): """Submit shell command to SGE queue via `qsub`""" - qsub_template = """echo {cmd} | qsub -o ":{outfile}" -e ":{errfile}" -V -r y -pe {pe} {n_cpu} -N {job_name}""" - return qsub_template.format(cmd=cmd, job_name=job_name, outfile=outfile, errfile=errfile, pe=pe, n_cpu=n_cpu) + submit_cmd = [ + "qsub", + "-o", + ":{outfile}".format(outfile=outfile), + "-e", + ":{errfile}".format(errfile=errfile), + "-V", + "-r", + "y", + "-pe", + str(pe), + str(n_cpu), + "-N", + str(job_name), + ] + return submit_cmd class SGEJobTask(luigi.Task): @@ -260,8 +274,8 @@ def _dump(self, out_dir=""): if self.__module__ == "__main__": d = pickle.dumps(self) module_name = os.path.basename(sys.argv[0]).rsplit(".", 1)[0] - d = d.replace("(c__main__", "(c" + module_name) - with open(self.job_file, "w") as f: + d = d.replace(b"(c__main__", b"(c" + module_name.encode("utf-8")) + with open(self.job_file, "wb") as f: f.write(d) else: with open(self.job_file, "wb") as f: @@ -281,10 +295,12 @@ def _run_job(self): self.outfile = os.path.join(self.tmp_dir, "job.out") self.errfile = os.path.join(self.tmp_dir, "job.err") submit_cmd = _build_qsub_command(job_str, self.task_family, self.outfile, self.errfile, self.parallel_env, self.n_cpu) - logger.debug("qsub command: \n" + submit_cmd) + logger.debug("qsub command: \n" + " ".join(submit_cmd)) # Submit the job and grab job ID - output = subprocess.check_output(submit_cmd, shell=True) + output = subprocess.check_output(submit_cmd, input=job_str.encode("utf-8")) + if isinstance(output, bytes): + output = output.decode("utf-8") self.job_id = _parse_qsub_job_id(output) logger.debug("Submitted job to qsub with response:\n" + output)