From 86d287f79a47061e3fa3f7426ec64fc998ea95eb Mon Sep 17 00:00:00 2001 From: virusdefender Date: Wed, 27 Jun 2018 16:35:58 +0800 Subject: [PATCH] add debug run api --- Dockerfile | 4 +-- client/Python/client.py | 15 ++++++++ server/config.py | 1 + server/entrypoint.sh | 2 +- server/judge_client.py | 40 ++++++++++++++++++++- server/server.py | 79 ++++++++++++++++++++++++++++++----------- 6 files changed, 116 insertions(+), 25 deletions(-) diff --git a/Dockerfile b/Dockerfile index 35ff25f..b623410 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,10 +3,10 @@ FROM registry.docker-cn.com/library/ubuntu:16.04 COPY build/java_policy /etc RUN buildDeps='software-properties-common git libtool cmake python-dev python3-pip python-pip libseccomp-dev' && \ - apt-get update && apt-get install -y python python3.5 python-pkg-resources python3-pkg-resources gcc g++ $buildDeps && \ + apt-get update && apt-get install -y python python3.5 python-pkg-resources python3-pkg-resources gcc g++ socat $buildDeps && \ add-apt-repository ppa:openjdk-r/ppa && apt-get update && apt-get install -y openjdk-8-jdk && \ pip3 install --no-cache-dir psutil gunicorn flask requests && \ - cd /tmp && git clone -b newnew --depth 1 https://github.com/QingdaoU/Judger && cd Judger && \ + cd /tmp && git clone -b feature/sock_redirect --depth 1 https://github.com/QingdaoU/Judger && cd Judger && \ mkdir build && cd build && cmake .. && make && make install && cd ../bindings/Python && python3 setup.py install && \ apt-get purge -y --auto-remove $buildDeps && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ diff --git a/client/Python/client.py b/client/Python/client.py index 4704240..c1eb81f 100644 --- a/client/Python/client.py +++ b/client/Python/client.py @@ -43,6 +43,16 @@ class JudgeServerClient(object): "output": output} return self._request(self.server_base_url + "/judge", data=data) + def debug_run(self, src, language_config, max_cpu_time, max_real_time, max_memory, submission_id): + data = {"language_config": language_config, + "src": src, + "max_cpu_time": max_cpu_time, + "max_real_time": max_real_time, + "max_memory": max_memory, + "submission_id": submission_id, + } + return self._request(self.server_base_url + "/debug_run", data=data) + def compile_spj(self, src, spj_version, spj_compile_config): data = {"src": src, "spj_version": spj_version, "spj_compile_config": spj_compile_config} @@ -111,6 +121,11 @@ print(int(s1[0]) + int(s1[1]))""" print(client.compile_spj(src=c_spj_src, spj_version="2", spj_compile_config=c_lang_spj_compile ), "\n\n") + # print(client.debug_run(src=c_src, language_config=c_lang_config, + # max_cpu_time=1000, max_real_time=30000, max_memory=1024 * 1024 * 128, + # submission_id="123"), "\n\n") + + print("c_judge") print(client.judge(src=c_src, language_config=c_lang_config, max_cpu_time=1000, max_memory=1024 * 1024 * 128, diff --git a/server/config.py b/server/config.py index 1e8a1ea..f94eb07 100644 --- a/server/config.py +++ b/server/config.py @@ -19,3 +19,4 @@ COMPILER_GROUP_GID = grp.getgrnam("compiler").gr_gid TEST_CASE_DIR = "/test_case" SPJ_SRC_DIR = "/judger/spj" SPJ_EXE_DIR = "/judger/spj" +IO_SOCK_DIR = "/judger/socks" diff --git a/server/entrypoint.sh b/server/entrypoint.sh index 817642d..642f1d9 100755 --- a/server/entrypoint.sh +++ b/server/entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/bash rm -rf /judger/* -mkdir -p /judger/run /judger/spj +mkdir -p /judger/run /judger/spj /judger/socks chown compiler:compiler /judger/spj core=$(grep --count ^processor /proc/cpuinfo) n=$(($core*2)) diff --git a/server/judge_client.py b/server/judge_client.py index 5f012d5..4874981 100644 --- a/server/judge_client.py +++ b/server/judge_client.py @@ -18,7 +18,45 @@ def _run(instance, test_case_file_id): return instance._judge_one(test_case_file_id) -class JudgeClient(object): +class DebugRunClient: + def __init__(self, run_config, exe_path, max_cpu_time, max_real_time, max_memory, submission_dir, io_sock_path): + self._run_config = run_config + self._exe_path = exe_path + self._max_cpu_time = max_cpu_time + self._max_real_time = max_real_time + self._max_memory = max_memory + self._submission_dir = submission_dir + self._io_sock_path = io_sock_path + + def _judge_one(self): + command = self._run_config["command"].format(exe_path=self._exe_path, exe_dir=os.path.dirname(self._exe_path), + max_memory=int(self._max_memory / 1024)).split(" ") + env = ["PATH=" + os.environ.get("PATH", "")] + self._run_config.get("env", []) + + run_result = _judger.run(max_cpu_time=self._max_cpu_time, + max_real_time=self._max_real_time, + max_memory=self._max_memory, + max_stack=128 * 1024 * 1024, + max_output_size=1024 * 1024 * 16, + max_process_number=_judger.UNLIMITED, + exe_path=command[0], + input_path=self._io_sock_path, + output_path=self._io_sock_path, + error_path=self._io_sock_path, + args=command[1::], + env=env, + log_path=JUDGER_RUN_LOG_PATH, + seccomp_rule_name=self._run_config["seccomp_rule"], + uid=RUN_USER_UID, + gid=RUN_GROUP_GID, + memory_limit_check_only=self._run_config.get("memory_limit_check_only", 0)) + return run_result + + def run(self): + return self._judge_one() + + +class JudgeClient: def __init__(self, run_config, exe_path, max_cpu_time, max_memory, test_case_id, submission_dir, spj_version, spj_config, output=False): self._run_config = run_config diff --git a/server/server.py b/server/server.py index c4fe300..46ca499 100644 --- a/server/server.py +++ b/server/server.py @@ -2,14 +2,16 @@ import json import os import shutil import uuid +import time + +from flask import Flask, Response, request -from flask import Flask, request, Response from compiler import Compiler -from config import JUDGER_WORKSPACE_BASE, SPJ_SRC_DIR, SPJ_EXE_DIR -from exception import TokenVerificationFailed, CompileError, SPJCompileError, JudgeClientError -from judge_client import JudgeClient -from utils import server_info, logger, token +from config import JUDGER_WORKSPACE_BASE, SPJ_EXE_DIR, SPJ_SRC_DIR, IO_SOCK_DIR +from exception import CompileError, JudgeClientError, SPJCompileError, TokenVerificationFailed +from judge_client import DebugRunClient, JudgeClient +from utils import logger, server_info, token app = Flask(__name__) DEBUG = os.environ.get("judger_debug") == "1" @@ -45,6 +47,55 @@ class JudgeServer: data["action"] = "pong" return data + @classmethod + def debug_run(cls, language_config, src, max_cpu_time, max_real_time, max_memory, submission_id): + # init + compile_config = language_config.get("compile") + run_config = language_config["run"] + # 等待 io sock 的建立 + wait = 10 + io_sock_path = os.path.join(IO_SOCK_DIR, submission_id) + while wait: + if not os.path.exists(io_sock_path): + time.sleep(1) + wait -= 1 + else: + break + else: + raise ValueError("IO Sock not found") + + with InitSubmissionEnv(JUDGER_WORKSPACE_BASE, submission_id=str(submission_id)) as submission_dir: + exe_path = cls._compile(submission_dir, compile_config=compile_config, run_config=run_config, src=src) + + judge_client = DebugRunClient(run_config=language_config["run"], + exe_path=exe_path, + max_cpu_time=max_cpu_time, + max_real_time=max_real_time, + max_memory=max_memory, + submission_dir=submission_dir, + io_sock_path="unix:" + io_sock_path) + run_result = judge_client.run() + return run_result + + @classmethod + def _compile(cls, submission_dir, compile_config, run_config, src): + if compile_config: + src_path = os.path.join(submission_dir, compile_config["src_name"]) + + # write source code into file + with open(src_path, "w", encoding="utf-8") as f: + f.write(src) + + # compile source code, return exe file path + exe_path = Compiler().compile(compile_config=compile_config, + src_path=src_path, + output_dir=submission_dir) + else: + exe_path = os.path.join(submission_dir, run_config["exe_name"]) + with open(exe_path, "w", encoding="utf-8") as f: + f.write(src) + return exe_path + @classmethod def judge(cls, language_config, src, max_cpu_time, max_memory, test_case_id, spj_version=None, spj_config=None, spj_compile_config=None, spj_src=None, output=False): @@ -62,21 +113,7 @@ class JudgeServer: spj_compile_config=spj_compile_config) with InitSubmissionEnv(JUDGER_WORKSPACE_BASE, submission_id=str(submission_id)) as submission_dir: - if compile_config: - src_path = os.path.join(submission_dir, compile_config["src_name"]) - - # write source code into file - with open(src_path, "w", encoding="utf-8") as f: - f.write(src) - - # compile source code, return exe file path - exe_path = Compiler().compile(compile_config=compile_config, - src_path=src_path, - output_dir=submission_dir) - else: - exe_path = os.path.join(submission_dir, run_config["exe_name"]) - with open(exe_path, "w", encoding="utf-8") as f: - f.write(src) + exe_path = cls._compile(submission_dir, compile_config=compile_config, run_config=run_config, src=src) judge_client = JudgeClient(run_config=language_config["run"], exe_path=exe_path, @@ -115,7 +152,7 @@ class JudgeServer: @app.route('/', defaults={'path': ''}) @app.route('/', methods=["POST"]) def server(path): - if path in ("judge", "ping", "compile_spj"): + if path in ("judge", "debug_run", "ping", "compile_spj"): _token = request.headers.get("X-Judge-Server-Token") try: if _token != token: