Python3 server

.travis.yml

@@ -6,8 +6,9 @@ before_install:
     - sudo mkdir -p /data/log
     - pip install requests
 script:
-    - docker pull qduoj/judge_server
+    - docker pull qduoj/judge-server
     - cp docker-compose.example.yml docker-compose.yml
     - docker-compose up -d
     - docker ps -a
+    - sleep 5
     - python tests/tests.py

Dockerfile

@@ -1,21 +1,20 @@
 FROM registry.docker-cn.com/library/ubuntu:16.04
-ENV DEBIAN_FRONTEND noninteractive
 
 COPY build/java_policy /etc
 
-RUN buildDeps='software-properties-common git libtool cmake python-dev python-pip libseccomp-dev' && \
-    apt-get update && apt-get install -y python python3.5 python-pkg-resources gcc g++ $buildDeps && \
-    add-apt-repository ppa:openjdk-r/ppa && apt-get update && apt-get install -y openjdk-7-jdk && \
-    pip install --no-cache-dir futures psutil gunicorn web.py requests && \
+RUN buildDeps='software-properties-common git libtool cmake python-dev python3-pip python-pip libseccomp-dev' && \
+    apt-get update && apt-get install -y python python3.5 python-pkg-resources python3-pkg-resources gcc g++ $buildDeps && \
+    add-apt-repository ppa:openjdk-r/ppa && apt-get update && apt-get install -y openjdk-9-jdk-headless && \
+    pip3 install --no-cache-dir psutil gunicorn flask requests && \
     cd /tmp && git clone -b newnew  --depth 1 https://github.com/QingdaoU/Judger && cd Judger && \ 
-    mkdir build && cd build && cmake .. && make && make install && cd ../bindings/Python && python setup.py install && \
+    mkdir build && cd build && cmake .. && make && make install && cd ../bindings/Python && python3 setup.py install && \
     apt-get purge -y --auto-remove $buildDeps && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /judger_run /test_case /log /code && \
     useradd -r compiler
 
-HEALTHCHECK --interval=5s --retries=3 CMD python /code/service.py
+HEALTHCHECK --interval=5s --retries=3 CMD python3 /code/service.py
 ADD server /code
 WORKDIR /code
 EXPOSE 8080

client/Python/client.py

@@ -1,6 +1,3 @@
-# coding=utf-8
-from __future__ import unicode_literals, print_function
-
 import hashlib
 import json
 
@@ -27,7 +24,7 @@ class JudgeServerClient(object):
         try:
             return requests.post(url, **kwargs).json()
         except Exception as e:
-            raise JudgeServerClientError(e.message)
+            raise JudgeServerClientError(str(e))
 
     def ping(self):
         return self._request(self.server_base_url + "/ping")

server/compiler.py

@@ -1,11 +1,7 @@
-# coding=utf-8
-from __future__ import unicode_literals
-
+import _judger
 import json
 import os
 
-import _judger
-
 from config import COMPILER_LOG_PATH, COMPILER_USER_UID, COMPILER_GROUP_GID
 from exception import CompileError
 
@@ -24,13 +20,13 @@ class Compiler(object):
                              max_stack=128 * 1024 * 1024,
                              max_output_size=1024 * 1024,
                              max_process_number=_judger.UNLIMITED,
-                             exe_path=_command[0].encode("utf-8"),
+                             exe_path=_command[0],
                              # /dev/null is best, but in some system, this will call ioctl system call
-                             input_path=src_path.encode("utf-8"),
-                             output_path=compiler_out.encode("utf-8"),
-                             error_path=compiler_out.encode("utf-8"),
-                             args=[item.encode("utf-8") for item in _command[1::]],
-                             env=[("PATH=" + os.getenv("PATH")).encode("utf-8")],
+                             input_path=src_path,
+                             output_path=compiler_out,
+                             error_path=compiler_out,
+                             args=_command[1::],
+                             env=["PATH=" + os.getenv("PATH")],
                              log_path=COMPILER_LOG_PATH,
                              seccomp_rule_name=None,
                              uid=COMPILER_USER_UID,
@@ -43,7 +39,7 @@ class Compiler(object):
                     os.remove(compiler_out)
                     if error:
                         raise CompileError(error)
-            raise CompileError("Compiler runtime error, info: %s" % json.dumps(result).decode("utf-8"))
+            raise CompileError("Compiler runtime error, info: %s" % json.dumps(result))
         else:
             os.remove(compiler_out)
             return exe_path

server/config.py

@@ -1,15 +1,14 @@
-# coding=utf-8
-from __future__ import unicode_literals
-
-import grp
 import os
 import pwd
 
+import grp
+
 JUDGER_WORKSPACE_BASE = "/judger_run"
 LOG_BASE = "/log"
 
-COMPILER_LOG_PATH = os.path.join(LOG_BASE, "compile.log").encode("utf-8")
-JUDGER_RUN_LOG_PATH = os.path.join(LOG_BASE, "judger.log").encode("utf-8")
+COMPILER_LOG_PATH = os.path.join(LOG_BASE, "compile.log")
+JUDGER_RUN_LOG_PATH = os.path.join(LOG_BASE, "judger.log")
+SERVER_LOG_PATH = os.path.join(LOG_BASE, "judge_server.log")
 
 RUN_USER_UID = pwd.getpwnam("nobody").pw_uid
 RUN_GROUP_GID = grp.getgrnam("nogroup").gr_gid

server/entrypoint.sh

@@ -2,4 +2,4 @@
 chown compiler:compiler /spj
 core=$(grep --count ^processor /proc/cpuinfo)
 n=$(($core*2))
-exec gunicorn --workers $n --threads $n --error-logfile /log/gunicorn.log --time 600 --bind 0.0.0.0:8080 server:wsgiapp
+exec gunicorn --workers $n --threads $n --error-logfile /log/gunicorn.log --time 600 --bind 0.0.0.0:8080 server:app

server/exception.py

@@ -1,21 +1,24 @@
-from __future__ import unicode_literals
+class JudgeServerException(Exception):
+    def __init__(self, message):
+        super().__init__()
+        self.message = message
 
 
-class CompileError(Exception):
+class CompileError(JudgeServerException):
     pass
 
 
-class SPJCompileError(CompileError):
+class SPJCompileError(JudgeServerException):
     pass
 
 
-class TokenVerificationFailed(Exception):
+class TokenVerificationFailed(JudgeServerException):
     pass
 
 
-class JudgeClientError(Exception):
+class JudgeClientError(JudgeServerException):
     pass
 
 
-class JudgeServiceError(Exception):
+class JudgeServiceError(JudgeServerException):
     pass

server/judge_client.py

@@ -1,17 +1,14 @@
-# coding=utf-8
-from __future__ import unicode_literals
 import _judger
-import psutil
-import os
-import json
 import hashlib
-
+import json
+import os
 from multiprocessing import Pool
 
+import psutil
+
 from config import TEST_CASE_DIR, JUDGER_RUN_LOG_PATH, RUN_GROUP_GID, RUN_USER_UID, SPJ_EXE_DIR
 from exception import JudgeClientError
 
-
 SPJ_WA = 1
 SPJ_AC = 0
 SPJ_ERROR = -1
@@ -40,14 +37,15 @@ class JudgeClient(object):
         self._spj_config = spj_config
         self._output = output
         if self._spj_version and self._spj_config:
-            self._spj_exe = os.path.join(SPJ_EXE_DIR, self._spj_config["exe_name"].format(spj_version=self._spj_version))
+            self._spj_exe = os.path.join(SPJ_EXE_DIR,
+                                         self._spj_config["exe_name"].format(spj_version=self._spj_version))
             if not os.path.exists(self._spj_exe):
                 raise JudgeClientError("spj exe not found")
 
     def _load_test_case_info(self):
         try:
             with open(os.path.join(self._test_case_dir, "info")) as f:
-                return json.loads(f.read())
+                return json.load(f)
         except IOError:
             raise JudgeClientError("Test case not found")
         except ValueError:
@@ -60,7 +58,7 @@ class JudgeClient(object):
         user_output_file = os.path.join(self._submission_dir, str(test_case_file_id) + ".out")
         with open(user_output_file, "r") as f:
             content = f.read()
-        output_md5 = hashlib.md5(content.rstrip()).hexdigest()
+        output_md5 = hashlib.md5(content.rstrip().encode("utf-8")).hexdigest()
         result = output_md5 == self._get_test_case_file_info(test_case_file_id)["stripped_output_md5"]
         return output_md5, result
 
@@ -68,19 +66,19 @@ class JudgeClient(object):
         command = self._spj_config["command"].format(exe_path=self._spj_exe,
                                                      in_file_path=in_file_path,
                                                      user_out_file_path=user_out_file_path).split(" ")
-        seccomp_rule_name = self._spj_config["seccomp_rule"].encode("utf-8") if self._spj_config["seccomp_rule"] else None
+        seccomp_rule_name = self._spj_config["seccomp_rule"]
         result = _judger.run(max_cpu_time=self._max_cpu_time * 3,
                              max_real_time=self._max_cpu_time * 9,
                              max_memory=self._max_memory * 3,
                              max_stack=128 * 1024 * 1024,
                              max_output_size=1024 * 1024 * 1024,
                              max_process_number=_judger.UNLIMITED,
-                             exe_path=command[0].encode("utf-8"),
-                             input_path=in_file_path.encode("utf-8"),
-                             output_path="/tmp/spj.out".encode("utf-8"),
-                             error_path="/tmp/spj.out".encode("utf-8"),
-                             args=[item.encode("utf-8") for item in command[1::]],
-                             env=[("PATH=" + os.environ.get("PATH", "")).encode("utf-8")],
+                             exe_path=command[0],
+                             input_path=in_file_path,
+                             output_path="/tmp/spj.out",
+                             error_path="/tmp/spj.out",
+                             args=command[1::],
+                             env=["PATH=" + os.environ.get("PATH", "")],
                              log_path=JUDGER_RUN_LOG_PATH,
                              seccomp_rule_name=seccomp_rule_name,
                              uid=RUN_USER_UID,
@@ -88,19 +86,18 @@ class JudgeClient(object):
 
         if result["result"] == _judger.RESULT_SUCCESS or \
                 (result["result"] == _judger.RESULT_RUNTIME_ERROR and
-                         result["exit_code"] in [SPJ_WA, SPJ_ERROR] and result["signal"] == 0):
+                 result["exit_code"] in [SPJ_WA, SPJ_ERROR] and result["signal"] == 0):
             return result["exit_code"]
         else:
             return SPJ_ERROR
 
     def _judge_one(self, test_case_file_id):
-        in_file = os.path.join(self._test_case_dir, self._get_test_case_file_info(test_case_file_id)["input_name"]).encode("utf-8")
-        user_output_file = os.path.join(self._submission_dir, test_case_file_id + ".out").encode("utf-8")
+        in_file = os.path.join(self._test_case_dir, self._get_test_case_file_info(test_case_file_id)["input_name"])
+        user_output_file = os.path.join(self._submission_dir, test_case_file_id + ".out")
 
         command = self._run_config["command"].format(exe_path=self._exe_path, exe_dir=os.path.dirname(self._exe_path),
                                                      max_memory=self._max_memory / 1024).split(" ")
-        seccomp_rule_name = self._run_config["seccomp_rule"].encode("utf-8") if self._run_config["seccomp_rule"] else None
-        env = [item.encode("utf-8") for item in ["PATH=" + os.environ.get("PATH", "")] + self._run_config.get("env", [])]
+        env = ["PATH=" + os.environ.get("PATH", "")] + self._run_config.get("env", [])
 
         run_result = _judger.run(max_cpu_time=self._max_cpu_time,
                                  max_real_time=self._max_real_time,
@@ -108,14 +105,14 @@ class JudgeClient(object):
                                  max_stack=128 * 1024 * 1024,
                                  max_output_size=1024 * 1024 * 1024,
                                  max_process_number=_judger.UNLIMITED,
-                                 exe_path=command[0].encode("utf-8"),
+                                 exe_path=command[0],
                                  input_path=in_file,
                                  output_path=user_output_file,
                                  error_path=user_output_file,
-                                 args=[item.encode("utf-8") for item in command[1::]],
+                                 args=command[1::],
                                  env=env,
                                  log_path=JUDGER_RUN_LOG_PATH,
-                                 seccomp_rule_name=seccomp_rule_name,
+                                 seccomp_rule_name=self._run_config["seccomp_rule"],
                                  uid=RUN_USER_UID,
                                  gid=RUN_GROUP_GID)
         run_result["test_case"] = test_case_file_id
@@ -144,7 +141,7 @@ class JudgeClient(object):
         if self._output:
             try:
                 with open(user_output_file, "r") as f:
-                    run_result["output"] = f.read().decode("utf-8")
+                    run_result["output"] = f.read()
             except Exception:
                 pass
 
@@ -153,7 +150,7 @@ class JudgeClient(object):
     def run(self):
         tmp_result = []
         result = []
-        for test_case_file_id, _ in self._test_case_info["test_cases"].iteritems():
+        for test_case_file_id, _ in self._test_case_info["test_cases"].items():
             tmp_result.append(self._pool.apply_async(_run, (self, test_case_file_id)))
         self._pool.close()
         self._pool.join()

server/server.py

@@ -1,21 +1,19 @@
-# coding=utf-8
-from __future__ import unicode_literals
-
 import json
 import os
 import shutil
 import uuid
 
-import web
+from flask import Flask, request, Response
 
 from compiler import Compiler
 from config import JUDGER_WORKSPACE_BASE, SPJ_SRC_DIR, SPJ_EXE_DIR
-from exception import TokenVerificationFailed, CompileError, SPJCompileError,JudgeClientError
+from exception import TokenVerificationFailed, CompileError, SPJCompileError, JudgeClientError
 from judge_client import JudgeClient
 from utils import server_info, logger, token
 
-
+app = Flask(__name__)
 DEBUG = os.environ.get("judger_debug") == "1"
+app.debug = DEBUG
 
 
 class InitSubmissionEnv(object):
@@ -25,7 +23,7 @@ class InitSubmissionEnv(object):
     def __enter__(self):
         try:
             os.mkdir(self.path)
-            os.chmod(self.path, 0777)
+            os.chmod(self.path, 0o777)
         except Exception as e:
             logger.exception(e)
             raise JudgeClientError("failed to create runtime dir")
@@ -40,26 +38,28 @@ class InitSubmissionEnv(object):
                 raise JudgeClientError("failed to clean runtime dir")
 
 
-class JudgeServer(object):
-    def pong(self):
+class JudgeServer:
+    @classmethod
+    def ping(cls):
         data = server_info()
         data["action"] = "pong"
         return data
 
-    def judge(self, language_config, src, max_cpu_time, max_memory, test_case_id,
+    @classmethod
+    def judge(cls, language_config, src, max_cpu_time, max_memory, test_case_id,
               spj_version=None, spj_config=None, spj_compile_config=None, spj_src=None, output=False):
         # init
         compile_config = language_config.get("compile")
         run_config = language_config["run"]
-        submission_id = str(uuid.uuid4())
+        submission_id = uuid.uuid4().hex
 
         if spj_version and spj_config:
             spj_exe_path = os.path.join(SPJ_EXE_DIR, spj_config["exe_name"].format(spj_version=spj_version))
             # spj src has not been compiled
             if not os.path.isfile(spj_exe_path):
                 logger.warning("%s does not exists, spj src will be recompiled")
-                self.compile_spj(spj_version=spj_version, src=spj_src,
-                             spj_compile_config=spj_compile_config)
+                cls.compile_spj(spj_version=spj_version, src=spj_src,
+                                spj_compile_config=spj_compile_config)
 
         with InitSubmissionEnv(JUDGER_WORKSPACE_BASE, submission_id=str(submission_id)) as submission_dir:
             if compile_config:
@@ -67,7 +67,7 @@ class JudgeServer(object):
 
                 # write source code into file
                 with open(src_path, "w") as f:
-                    f.write(src.encode("utf-8"))
+                    f.write(src)
 
                 # compile source code, return exe file path
                 exe_path = Compiler().compile(compile_config=compile_config,
@@ -76,7 +76,7 @@ class JudgeServer(object):
             else:
                 exe_path = os.path.join(submission_dir, run_config["exe_name"])
                 with open(exe_path, "w") as f:
-                    f.write(src.encode("utf-8"))
+                    f.write(src)
 
             judge_client = JudgeClient(run_config=language_config["run"],
                                        exe_path=exe_path,
@@ -91,7 +91,8 @@ class JudgeServer(object):
 
             return run_result
 
-    def compile_spj(self, spj_version, src, spj_compile_config):
+    @classmethod
+    def compile_spj(cls, spj_version, src, spj_compile_config):
         spj_compile_config["src_name"] = spj_compile_config["src_name"].format(spj_version=spj_version)
         spj_compile_config["exe_name"] = spj_compile_config["exe_name"].format(spj_version=spj_version)
 
@@ -100,7 +101,7 @@ class JudgeServer(object):
         # if spj source code not found, then write it into file
         if not os.path.exists(spj_src_path):
             with open(spj_src_path, "w") as f:
-                f.write(src.encode("utf-8"))
+                f.write(src)
         try:
             Compiler().compile(compile_config=spj_compile_config,
                                src_path=spj_src_path,
@@ -110,56 +111,34 @@ class JudgeServer(object):
             raise SPJCompileError(e.message)
         return "success"
 
-    def POST(self):
-        _token = web.ctx.env.get("HTTP_X_JUDGE_SERVER_TOKEN", None)
+
+@app.route('/', defaults={'path': ''})
+@app.route('/<path:path>', methods=["POST"])
+def server(path):
+    if path in ("judge", "ping", "compile_spj"):
+        _token = request.headers.get("X-Judge-Server-Token")
         try:
             if _token != token:
                 raise TokenVerificationFailed("invalid token")
-            if web.data():
-                try:
-                    data = json.loads(web.data())
-                except Exception as e:
-                    logger.info(web.data())
-                    return {"ret": "ServerError", "data": "invalid json"}
-            else:
+            try:
+                data = request.json
+            except Exception:
                 data = {}
-
-            if web.ctx["path"] == "/judge":
-                callback = self.judge
-            elif web.ctx["path"] == "/ping":
-                callback = self.pong
-            elif web.ctx["path"] == "/compile_spj":
-                callback = self.compile_spj
-            else:
-                return json.dumps({"err": "InvalidMethod", "data": None})
-            return json.dumps({"err": None, "data": callback(**data)})
+            ret = {"err": None, "data": getattr(JudgeServer, path)(**data)}
         except (CompileError, TokenVerificationFailed, SPJCompileError, JudgeClientError) as e:
             logger.exception(e)
-            ret = dict()
-            ret["err"] = e.__class__.__name__
-            ret["data"] = e.message
-            return json.dumps(ret)
+            ret = {"err": e.__class__.__name__, "data": e.message}
         except Exception as e:
             logger.exception(e)
-            ret = dict()
-            ret["err"] = "JudgeClientError"
-            ret["data"] =e.__class__.__name__ + ":" + e.message
-            return json.dumps(ret)
-
-
-urls = (
-    "/judge", "JudgeServer",
-    "/ping", "JudgeServer",
-    "/compile_spj", "JudgeServer"
-)
+            ret = {"err": "JudgeClientError", "data": e.__class__.__name__ + " :" + str(e)}
+    else:
+        ret = {"err": "InvalidRequest", "data": "404"}
+    return Response(json.dumps(ret), mimetype='application/json')
 
 
 if DEBUG:
     logger.info("DEBUG=ON")
 
-app = web.application(urls, globals())
-wsgiapp = app.wsgifunc()
-
-# gunicorn -w 4 -b 0.0.0.0:8080 server:wsgiapp
+# gunicorn -w 4 -b 0.0.0.0:8080 server:app
 if __name__ == "__main__":
-    app.run()
+    app.run(debug=DEBUG)

server/service.py

@@ -1,9 +1,7 @@
-# coding=utf-8
-from __future__ import unicode_literals
-import os
 import json
+import os
+
 import requests
-import hashlib
 
 from exception import JudgeServiceError
 from utils import server_info, logger, token
@@ -30,7 +28,6 @@ class JudgeService(object):
             logger.exception("Heartbeat failed, response is {}".format(resp))
             raise JudgeServiceError("Invalid heartbeat response")
 
-
     def heartbeat(self):
         data = server_info()
         data["action"] = "heartbeat"

server/utils.py

@@ -1,17 +1,16 @@
-# coding=utf-8
-from __future__ import unicode_literals
 import _judger
-import psutil
-import socket
-import logging
 import hashlib
+import logging
 import os
+import socket
 
+import psutil
+
+from config import SERVER_LOG_PATH
 from exception import JudgeClientError
 
-
 logger = logging.getLogger(__name__)
-handler = logging.FileHandler("/log/judge_server.log")
+handler = logging.FileHandler(SERVER_LOG_PATH)
 formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
 handler.setFormatter(formatter)
 logger.addHandler(handler)
@@ -32,8 +31,7 @@ def get_token():
     if token:
         return token
     else:
-        raise JudgeClientError("env 'token' not found")
+        raise JudgeClientError("env 'TOKEN' not found")
 
 
-token = hashlib.sha256(get_token()).hexdigest()
-
+token = hashlib.sha256(get_token().encode("utf-8")).hexdigest()