diff --git a/Dockerfile b/Dockerfile index 112ebb1..21d1e9b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ RUN buildDeps='software-properties-common git libtool cmake python-dev python3-p apt-get update && apt-get install -y python python3.5 python-pkg-resources python3-pkg-resources gcc g++ $buildDeps && \ add-apt-repository ppa:openjdk-r/ppa && apt-get update && apt-get install -y openjdk-8-jdk && \ pip3 install --no-cache-dir psutil gunicorn flask requests && \ - cd /tmp && git clone -b newnew --depth 1 https://github.com/QingdaoU/Judger && cd Judger && \ + cd /tmp && git clone -b newnew --depth 1 https://github.com/QingdaoU/Judger && cd Judger && \ mkdir build && cd build && cmake .. && make && make install && cd ../bindings/Python && python3 setup.py install && \ apt-get purge -y --auto-remove $buildDeps && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ @@ -16,5 +16,6 @@ RUN buildDeps='software-properties-common git libtool cmake python-dev python3-p HEALTHCHECK --interval=5s --retries=3 CMD python3 /code/service.py ADD server /code WORKDIR /code +RUN gcc -shared -fPIC -o unbuffer.so unbuffer.c EXPOSE 8080 ENTRYPOINT /code/entrypoint.sh diff --git a/client/Python/client.py b/client/Python/client.py index 4704240..4609233 100644 --- a/client/Python/client.py +++ b/client/Python/client.py @@ -29,13 +29,17 @@ class JudgeServerClient(object): def ping(self): return self._request(self.server_base_url + "/ping") - def judge(self, src, language_config, max_cpu_time, max_memory, test_case_id, spj_version=None, spj_config=None, + def judge(self, src, language_config, max_cpu_time, max_memory, test_case_id=None, test_case=None, spj_version=None, spj_config=None, spj_compile_config=None, spj_src=None, output=False): + if not (test_case or test_case_id) or (test_case and test_case_id): + raise ValueError("invalid parameter") + data = {"language_config": language_config, "src": src, "max_cpu_time": max_cpu_time, "max_memory": max_memory, "test_case_id": test_case_id, + "test_case": test_case, "spj_version": spj_version, "spj_config": spj_config, "spj_compile_config": spj_compile_config, @@ -143,3 +147,8 @@ print(int(s1[0]) + int(s1[1]))""" print(client.judge(src=py3_src, language_config=py3_lang_config, max_cpu_time=1000, max_memory=128 * 1024 * 1024, test_case_id="normal"), "\n\n") + + print("c_dynamic_input_judge") + print(client.judge(src=c_src, language_config=c_lang_config, + max_cpu_time=1000, max_memory=1024 * 1024 * 128, + test_case=[{"input": "1 2\n", "output": "3"}, {"input": "1 4\n", "output": "3"}], output=True), "\n\n") diff --git a/server/judge_client.py b/server/judge_client.py index b1de457..98a76cb 100644 --- a/server/judge_client.py +++ b/server/judge_client.py @@ -19,15 +19,14 @@ def _run(instance, test_case_file_id): class JudgeClient(object): - def __init__(self, run_config, exe_path, max_cpu_time, max_memory, test_case_id, + def __init__(self, run_config, exe_path, max_cpu_time, max_memory, test_case_dir, submission_dir, spj_version, spj_config, output=False): self._run_config = run_config self._exe_path = exe_path self._max_cpu_time = max_cpu_time self._max_memory = max_memory self._max_real_time = self._max_cpu_time * 3 - self._test_case_id = test_case_id - self._test_case_dir = os.path.join(TEST_CASE_DIR, test_case_id) + self._test_case_dir = test_case_dir self._submission_dir = submission_dir self._pool = Pool(processes=psutil.cpu_count()) @@ -145,8 +144,8 @@ class JudgeClient(object): if self._output: try: - with open(user_output_file, "r", encoding="utf-8") as f: - run_result["output"] = f.read() + with open(user_output_file, "rb") as f: + run_result["output"] = f.read().decode("utf-8", errors="backslashreplace") except Exception: pass diff --git a/server/server.py b/server/server.py index a07b30f..6a24631 100644 --- a/server/server.py +++ b/server/server.py @@ -1,3 +1,4 @@ +import hashlib import json import os import shutil @@ -6,7 +7,8 @@ import uuid from flask import Flask, request, Response from compiler import Compiler -from config import JUDGER_WORKSPACE_BASE, SPJ_SRC_DIR, SPJ_EXE_DIR, COMPILER_USER_UID, SPJ_USER_UID, RUN_USER_UID, RUN_GROUP_GID +from config import (JUDGER_WORKSPACE_BASE, SPJ_SRC_DIR, SPJ_EXE_DIR, COMPILER_USER_UID, SPJ_USER_UID, + RUN_USER_UID, RUN_GROUP_GID, TEST_CASE_DIR) from exception import TokenVerificationFailed, CompileError, SPJCompileError, JudgeClientError from judge_client import JudgeClient from utils import server_info, logger, token @@ -17,23 +19,30 @@ app.debug = DEBUG class InitSubmissionEnv(object): - def __init__(self, judger_workspace, submission_id): - self.path = os.path.join(judger_workspace, submission_id) + def __init__(self, judger_workspace, submission_id, init_test_case_dir=False): + self.work_dir = os.path.join(judger_workspace, submission_id) + self.init_test_case_dir = init_test_case_dir + if init_test_case_dir: + self.test_case_dir = os.path.join(self.work_dir, "submission_" + submission_id) + else: + self.test_case_dir = None def __enter__(self): try: - os.mkdir(self.path) - os.chown(self.path, COMPILER_USER_UID, RUN_GROUP_GID) - os.chmod(self.path, 0o711) + os.mkdir(self.work_dir) + if self.init_test_case_dir: + os.mkdir(self.test_case_dir) + os.chown(self.work_dir, COMPILER_USER_UID, RUN_GROUP_GID) + os.chmod(self.work_dir, 0o711) except Exception as e: logger.exception(e) raise JudgeClientError("failed to create runtime dir") - return self.path + return self.work_dir, self.test_case_dir def __exit__(self, exc_type, exc_val, exc_tb): if not DEBUG: try: - shutil.rmtree(self.path) + shutil.rmtree(self.work_dir) except Exception as e: logger.exception(e) raise JudgeClientError("failed to clean runtime dir") @@ -47,14 +56,18 @@ class JudgeServer: return data @classmethod - def judge(cls, language_config, src, max_cpu_time, max_memory, test_case_id, + def judge(cls, language_config, src, max_cpu_time, max_memory, test_case_id=None, test_case=None, spj_version=None, spj_config=None, spj_compile_config=None, spj_src=None, output=False): + if not (test_case or test_case_id) or (test_case and test_case_id): + raise JudgeClientError("invalid parameter") # init compile_config = language_config.get("compile") run_config = language_config["run"] submission_id = uuid.uuid4().hex - if spj_version and spj_config: + is_spj = spj_version and spj_config + + if is_spj: spj_exe_path = os.path.join(SPJ_EXE_DIR, spj_config["exe_name"].format(spj_version=spj_version)) # spj src has not been compiled if not os.path.isfile(spj_exe_path): @@ -62,7 +75,11 @@ class JudgeServer: cls.compile_spj(spj_version=spj_version, src=spj_src, spj_compile_config=spj_compile_config) - with InitSubmissionEnv(JUDGER_WORKSPACE_BASE, submission_id=str(submission_id)) as submission_dir: + init_test_case_dir = bool(test_case) + with InitSubmissionEnv(JUDGER_WORKSPACE_BASE, submission_id=str(submission_id), init_test_case_dir=init_test_case_dir) as dirs: + submission_dir, test_case_dir = dirs + test_case_dir = test_case_dir or os.path.join(TEST_CASE_DIR, test_case_id) + if compile_config: src_path = os.path.join(submission_dir, compile_config["src_name"]) @@ -88,11 +105,39 @@ class JudgeServer: with open(exe_path, "w", encoding="utf-8") as f: f.write(src) + if init_test_case_dir: + info = {"test_case_number": len(test_case), "spj": is_spj, "test_cases": {}} + # write test case + for index, item in enumerate(test_case): + index += 1 + item_info = {} + + input_name = str(index) + ".in" + item_info["input_name"] = input_name + input_data = item["input"].encode("utf-8") + item_info["input_size"] = len(input_data) + + with open(os.path.join(test_case_dir, input_name), "wb") as f: + f.write(input_data) + if not is_spj: + output_name = str(index) + ".out" + item_info["output_name"] = output_name + output_data = item["output"].encode("utf-8") + item_info["output_md5"] = hashlib.md5(output_data).hexdigest() + item_info["output_size"] = len(output_data) + item_info["stripped_output_md5"] = hashlib.md5(output_data.rstrip()).hexdigest() + + with open(os.path.join(test_case_dir, output_name), "wb") as f: + f.write(output_data) + info["test_cases"][index] = item_info + with open(os.path.join(test_case_dir, "info"), "w") as f: + json.dump(info, f) + judge_client = JudgeClient(run_config=language_config["run"], exe_path=exe_path, max_cpu_time=max_cpu_time, max_memory=max_memory, - test_case_id=str(test_case_id), + test_case_dir=test_case_dir, submission_dir=submission_dir, spj_version=spj_version, spj_config=spj_config, diff --git a/server/service.py b/server/service.py index fda002f..ae90898 100644 --- a/server/service.py +++ b/server/service.py @@ -37,8 +37,9 @@ class JudgeService(object): if __name__ == "__main__": try: - service = JudgeService() - service.heartbeat() + if not os.environ.get("DISABLE_HEARTBEAT"): + service = JudgeService() + service.heartbeat() exit(0) except Exception as e: logger.exception(e) diff --git a/server/unbuffer.c b/server/unbuffer.c new file mode 100644 index 0000000..919f2af --- /dev/null +++ b/server/unbuffer.c @@ -0,0 +1,8 @@ +#include + +void unbuffer() __attribute__((constructor)); + +void unbuffer() +{ + setvbuf(stdout, NULL, _IONBF, 0); +} \ No newline at end of file