From cc4b75edf5dcf901141e862c64af5ceef815ae4b Mon Sep 17 00:00:00 2001 From: LiYang Date: Fri, 2 Sep 2016 20:04:29 +0800 Subject: [PATCH] init proj --- .gitignore | 72 ++++++++++++++++++++++++++++++++ Dockerfile | 5 +++ LICENSE | 37 +++++++++++++++++ README.md | 3 ++ __init__.py | 0 client.py | 85 ++++++++++++++++++++++++++++++++++++++ compiler.py | 46 +++++++++++++++++++++ config.py | 16 ++++++++ exception.py | 13 ++++++ server.py | 113 +++++++++++++++++++++++++++++++++++++++++++++++++++ utils.py | 24 +++++++++++ 11 files changed, 414 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 README.md create mode 100644 __init__.py create mode 100644 client.py create mode 100644 compiler.py create mode 100644 config.py create mode 100644 exception.py create mode 100644 server.py create mode 100644 utils.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2f53043 --- /dev/null +++ b/.gitignore @@ -0,0 +1,72 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +# C extensions +*.so +# Distribution / packaging +.Python +env/ +bin/ +build/ +develop-eggs/ +dist/ +eggs/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg +# Installer logs +pip-log.txt +pip-delete-this-directory.txt +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml +# Translations +*.mo +# Mr Developer +.mr.developer.cfg +.project +.pydevproject +# Rope +.ropeproject +.idea/ +# Django stuff: +*.log +*.pot +# Sphinx documentation +docs/_build/ +# Back up file +*~ +#db file +db.db +#vim cache +*swp +*swo +#redis dump +*.rdb +#*.out +*.sqlite3 +.DS_Store +log/ +static/release/css +static/release/js +static/release/img +static/src/upload_image/* +build.txt +tmp/ +test_case/ +release/ +upload/ +custom_settings.py +docker-compose.yml +*.zip +rsyncd.passwd + +node_modules/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..34d04fd --- /dev/null +++ b/Dockerfile @@ -0,0 +1,5 @@ +FROM judger +RUN apt-get update && apt-get install -y cmake vim +RUN cd /tmp && rm -rf pyfadeaway && git clone https://github.com/nikoloss/pyfadeaway.git && cd pyfadeaway && python setup.py install +RUN cd /tmp && rm -rf Judger && git clone https://github.com/QingdaoU/Judger.git && cd Judger && git checkout newnew && mkdir build && cd build && cmake .. && make && make install && cd ../bindings/Python && python setup.py install +RUN pip install psutil \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7e40488 --- /dev/null +++ b/LICENSE @@ -0,0 +1,37 @@ +The Star And Thank Author License (SATA) + +Copyright (c) + +Project Url: https://github.com/QingdaoU/OnlineJudge + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +And wait, the most important, you shall star/+1/like the project(s) in project url +section above first, and then thank the author(s) in Copyright section. + +Here are some suggested ways: + + - Email the authors a thank-you letter, and make friends with him/her/them. + - Report bugs or issues. + - Tell friends what a wonderful project this is. + - And, sure, you can just express thanks in your mind without telling the world. + +Contributors of this project by forking have the option to add his/her name and +forked project url at copyright and project url sections, but shall not delete +or modify anything else in these two sections. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..23265d3 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# JudgeServer + +RPC backend for online judge judge server \ No newline at end of file diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/client.py b/client.py new file mode 100644 index 0000000..559f53c --- /dev/null +++ b/client.py @@ -0,0 +1,85 @@ +# coding=utf-8 +import hashlib +import httplib +import json +import time +import xmlrpclib + +from utils import make_signature, check_signature + + +class TimeoutHTTPConnection(httplib.HTTPConnection): + def __init__(self, host, timeout=10): + httplib.HTTPConnection.__init__(self, host, timeout=timeout) + + +class TimeoutTransport(xmlrpclib.Transport): + def __init__(self, timeout=10, *args, **kwargs): + xmlrpclib.Transport.__init__(self, *args, **kwargs) + self.timeout = timeout + + def make_connection(self, host): + conn = TimeoutHTTPConnection(host, self.timeout) + return conn + + +class TimeoutServerProxy(xmlrpclib.ServerProxy): + def __init__(self, uri, timeout=10, *args, **kwargs): + kwargs['transport'] = TimeoutTransport(timeout=timeout, use_datetime=kwargs.get('use_datetime', 0)) + xmlrpclib.ServerProxy.__init__(self, uri, *args, **kwargs) + + +c_lang_config = { + "name": "c", + "compile": { + "src_name": "main.c", + "exe_name": "main", + "max_cpu_time": 3000, + "max_real_time": 5000, + "max_memory": 128 * 1024 * 1024, + "compile_command": "/usr/bin/gcc -DONLINE_JUDGE -O2 -w -fmax-errors=3 -std=c99 -static {src_path} -lm -o {exe_path}", + }, + "spj_compile": { + "src_name": "spj-%s.c", + "exe_name": "spj-%s", + "max_cpu_time": 10000, + "max_real_time": 20000, + "max_memory": 1024 * 1024 * 1024, + "compile_command": "/usr/bin/gcc -DONLINE_JUDGE -O2 -w -fmax-errors=3 -std=c99 -static {src_path} -lm -o {exe_path}", + # server should replace to real info + "version": "1", + "src": "" + } +} + +submission_id = str(int(time.time())) +spj_config = c_lang_config["spj_compile"] + +s = TimeoutServerProxy("http://192.168.99.100:8080", timeout=30, allow_none=True) + +config = c_lang_config +config["spj_compile"]["version"] = "1024" +config["spj_compile"]["src"] = "#include\nint main(){//哈哈哈哈\nreturn 0;}" + +token = hashlib.sha256("token").hexdigest() + + +def pong(): + data, signature, timestamp = s.pong() + check_signature(token=token, data=data, signature=signature, timestamp=timestamp) + print json.loads(data) + + +def judge(): + data, signature, timestamp = s.judge(*make_signature(token=token, + language_config=c_lang_config, + submission_id=submission_id, + src="#include\nint main(){//哈哈哈哈\nreturn 0;}", + time_limit=1000, memory_limit=1000, test_case_id="2")) + + check_signature(token=token, data=data, signature=signature, timestamp=timestamp) + print json.loads(data) + + +pong() +judge() \ No newline at end of file diff --git a/compiler.py b/compiler.py new file mode 100644 index 0000000..2fba711 --- /dev/null +++ b/compiler.py @@ -0,0 +1,46 @@ +# coding=utf-8 +from __future__ import unicode_literals + +import json +import os + +import _judger + +from config import COMPILER_LOG_PATH, LOW_PRIVILEDGE_UID, LOW_PRIVILEDGE_GID +from exception import CompileError + + +class Compiler(object): + def compile(self, compile_config, src_path, output_dir): + command = compile_config["compile_command"] + exe_path = os.path.join(output_dir, compile_config["exe_name"]) + command = command.format(src_path=src_path, exe_path=exe_path) + compiler_out = os.path.join(output_dir, "compiler.out") + _command = command.split(" ") + + result = _judger.run(max_cpu_time=compile_config["max_cpu_time"], + max_real_time=compile_config["max_real_time"], + max_memory=compile_config["max_memory"], + max_output_size=1024 * 1024, + max_process_number=20, + exe_path=_command[0], + # /dev/null is best, but in some system, this will call ioctl system call + input_path=src_path, + output_path=compiler_out, + error_path=compiler_out, + args=[item.encode("utf-8") for item in _command[1::]], + env=[("PATH=" + os.getenv("PATH")).encode("utf-8")], + log_path=COMPILER_LOG_PATH, + seccomp_rule_so_path=None, + uid=LOW_PRIVILEDGE_UID, + gid=LOW_PRIVILEDGE_GID) + + if result["result"] != _judger.RESULT_SUCCESS: + with open(compiler_out) as f: + error = f.read().strip() + if error: + raise CompileError(error) + + raise CompileError("Compiler runtime error, info: %s" % json.dumps(result).decode("utf-8")) + + return exe_path diff --git a/config.py b/config.py new file mode 100644 index 0000000..6d796e3 --- /dev/null +++ b/config.py @@ -0,0 +1,16 @@ +# coding=utf-8 +from __future__ import unicode_literals + +import grp +import os +import pwd + +JUDGER_WORKSPACE_BASE = "/var/wp" + +COMPILER_LOG_PATH = os.path.join(JUDGER_WORKSPACE_BASE, "compile.log") +JUDGER_RUN_LOG_PATH = os.path.join(JUDGER_WORKSPACE_BASE, "judger.log") + +LOW_PRIVILEDGE_UID = pwd.getpwnam("nobody").pw_uid +LOW_PRIVILEDGE_GID = grp.getgrnam("nogroup").gr_gid + +TEST_CASE_DIR = "/var/testcase" diff --git a/exception.py b/exception.py new file mode 100644 index 0000000..b3c6379 --- /dev/null +++ b/exception.py @@ -0,0 +1,13 @@ +from __future__ import unicode_literals + + +class CompileError(Exception): + pass + + +class SPJCompileError(CompileError): + pass + + +class SignatureVerificationFailed(Exception): + pass diff --git a/server.py b/server.py new file mode 100644 index 0000000..70c385c --- /dev/null +++ b/server.py @@ -0,0 +1,113 @@ +# coding=utf-8 +from __future__ import unicode_literals + +import SocketServer +import hashlib +import json +import os +import socket +import time +from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler + +import _judger +import psutil + +from compiler import Compiler +from config import JUDGER_WORKSPACE_BASE, TEST_CASE_DIR +from exception import SignatureVerificationFailed, CompileError, SPJCompileError +from utils import make_signature, check_signature + + +class InitSubmissionEnv(object): + def __init__(self, judger_workspace, submission_id): + self.path = os.path.join(judger_workspace, submission_id) + + def __enter__(self): + os.mkdir(self.path) + os.chmod(self.path, 0777) + return self.path + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + # shutil.rmtree(self.path, ignore_errors=True) + + +class JudgeServer(object): + def health_check(self): + ver = _judger.VERSION + return {"hostname": socket.gethostname(), + "cpu": psutil.cpu_percent(), + "memory": psutil.virtual_memory().percent, + "judger_version": ((ver >> 16) & 0xff, (ver >> 8) & 0xff, ver & 0xff)} + + def pong(self): + return make_signature(token=self.token, **self.health_check()) + + @property + def token(self): + t = os.getenv("judger_token") + if not t: + raise SignatureVerificationFailed("token not set") + return hashlib.sha256(t).hexdigest() + + def judge(self, data, signature, timestamp): + check_signature(token=self.token, data=data, signature=signature, timestamp=timestamp) + ret = {"code": None, "data": None} + try: + ret["data"] = self._judge(**json.loads(data)) + except (CompileError, SPJCompileError, SignatureVerificationFailed) as e: + ret["code"] = e.__class__.__name__ + ret["data"] = e.message + except Exception as e: + ret["code"] = "ServerError" + ret["data"] = e.message + return make_signature(token=self.token, **ret) + + def _judge(self, language_config, submission_id, src, time_limit, memory_limit, test_case_id): + # init + compile_config = language_config["compile"] + spj_compile_config = language_config.get("spj_compile") + + with InitSubmissionEnv(JUDGER_WORKSPACE_BASE, submission_id=submission_id) as submission_dir: + src_path = os.path.join(submission_dir, compile_config["src_name"]) + + # write source code into file + with open(src_path, "w") as f: + f.write(src.encode("utf-8")) + + # compile source code, return exe file path + exe_path = Compiler().compile(compile_config=compile_config, + src_path=src_path, + output_dir=submission_dir) + + if spj_compile_config: + spj_compile_config["src_name"] %= spj_compile_config["version"] + spj_compile_config["exe_name"] %= spj_compile_config["version"] + + spj_src_path = os.path.join(TEST_CASE_DIR, test_case_id, spj_compile_config["src_name"]) + + # if spj source code not found, then write it into file + if not os.path.exists(spj_src_path): + with open(spj_src_path, "w") as f: + f.write(spj_compile_config["src"].encode("utf-8")) + + spj_exe_path = os.path.join(TEST_CASE_DIR, test_case_id, spj_compile_config["exe_name"]) + + # if spj exe file not found, then compile it + if not os.path.exists(spj_exe_path): + try: + spj_exe_path = Compiler().compile(compile_config=spj_compile_config, + src_path=spj_src_path, + output_dir=os.path.join(TEST_CASE_DIR, test_case_id)) + # turn common CompileError into SPJCompileError + except CompileError as e: + raise SPJCompileError(e.message) + + +class AsyncXMLRPCServer(SocketServer.ThreadingMixIn, SimpleXMLRPCServer): + pass + + +server = AsyncXMLRPCServer(('0.0.0.0', 8080), SimpleXMLRPCRequestHandler, allow_none=True) +server.register_instance(JudgeServer()) +server.serve_forever() diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..1be935f --- /dev/null +++ b/utils.py @@ -0,0 +1,24 @@ +# coding=utf-8 +from __future__ import unicode_literals + +import json +import time +import hashlib + +from exception import SignatureVerificationFailed + + +def make_signature(**kwargs): + token = kwargs.pop("token") + data = json.dumps(kwargs) + timestamp = int(time.time()) + return data, hashlib.sha256(data + str(timestamp) + token).hexdigest(), timestamp + + +def check_signature(token, data, signature, timestamp): + ts = int(time.time()) + if abs(timestamp - ts) > 5: + raise SignatureVerificationFailed("Timestamp interval is too long") + + if hashlib.sha256(data + str(timestamp) + token).hexdigest() != signature: + raise SignatureVerificationFailed("Wrong signature") \ No newline at end of file