From efddadb89892d84f60b43c5d53c55e8fa4d3f439 Mon Sep 17 00:00:00 2001 From: virusdefender Date: Sat, 5 Jan 2019 13:13:06 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E5=88=9D=E5=A7=8B=E5=8C=96=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + .travis.yml | 6 +----- init_db.sh | 19 +++++++++++++++++++ oj/dev_settings.py | 4 ++-- 4 files changed, 23 insertions(+), 7 deletions(-) create mode 100755 init_db.sh diff --git a/.gitignore b/.gitignore index d4b59810..8b3b73ff 100644 --- a/.gitignore +++ b/.gitignore @@ -76,3 +76,4 @@ data/config/* http_locations.conf https_locations.conf +venv/ diff --git a/.travis.yml b/.travis.yml index 9edf0294..ac84fa0a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,15 +2,11 @@ language: python python: - "3.6" services: - - redis-server - docker -before_install: - - docker pull postgres:10 - - docker run -it -d -e POSTGRES_DB=onlinejudge -e POSTGRES_USER=onlinejudge -e POSTGRES_PASSWORD=onlinejudge -p 127.0.0.1:5433:5432 postgres:10 install: - pip install -r deploy/requirements.txt - echo `cat /dev/urandom | head -1 | md5sum | head -c 32` > data/config/secret.key - - sleep 10 && python manage.py migrate + - ./init_db.sh script: - docker ps -a - flake8 . diff --git a/init_db.sh b/init_db.sh new file mode 100755 index 00000000..6fc29b06 --- /dev/null +++ b/init_db.sh @@ -0,0 +1,19 @@ +#! /bin/bash +set -x + +if [[ ! -f manage.py ]]; then + echo "No manage.py, wrong location" + exit 1 +fi + +sleep 2 +docker rm -f oj-postgres-dev oj-redis-dev +docker run -it -d -e POSTGRES_DB=onlinejudge -e POSTGRES_USER=onlinejudge -e POSTGRES_PASSWORD=onlinejudge -p 127.0.0.1:5435:5432 --name oj-postgres-dev postgres:10 +docker run -it -d -p 127.0.0.1:6380:6379 --name oj-redis-dev redis:4.0-alpine + +if [ "$1" = "--migrate" ]; then + sleep 3 + echo `cat /dev/urandom | head -1 | md5sum | head -c 32` > data/config/secret.key + python manage.py migrate + python manage.py inituser --username root --password rootroot --action create_super_admin +fi \ No newline at end of file diff --git a/oj/dev_settings.py b/oj/dev_settings.py index 5c75a0b2..4736c69f 100644 --- a/oj/dev_settings.py +++ b/oj/dev_settings.py @@ -7,7 +7,7 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'HOST': '127.0.0.1', - 'PORT': 5433, + 'PORT': 5435, 'NAME': "onlinejudge", 'USER': "onlinejudge", 'PASSWORD': 'onlinejudge' @@ -16,7 +16,7 @@ DATABASES = { REDIS_CONF = { "host": "127.0.0.1", - "port": "6379" + "port": "6380" } From 2dc1d0cbce59c2983699d83fbd65cea77bcc9d1c Mon Sep 17 00:00:00 2001 From: virusdefender Date: Sat, 5 Jan 2019 13:13:30 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20ACM=20=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=E9=A2=98=E7=9B=AE=E5=AF=BC=E5=85=A5=E5=90=8E=EF=BC=8C?= =?UTF-8?q?=E6=97=A0=E6=B3=95=E6=98=BE=E7=A4=BA=E6=B5=8B=E8=AF=95=E7=94=A8?= =?UTF-8?q?=E4=BE=8B=E4=BF=A1=E6=81=AF=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- problem/serializers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/problem/serializers.py b/problem/serializers.py index dbea96b7..de9bfdf6 100644 --- a/problem/serializers.py +++ b/problem/serializers.py @@ -154,8 +154,9 @@ class ExportProblemSerializer(serializers.ModelSerializer): return self._html_format_value(obj.hint) def get_test_case_score(self, obj): - return [{"score": item["score"], "input_name": item["input_name"], "output_name": item["output_name"]} - for item in obj.test_case_score] if obj.rule_type == ProblemRuleType.OI else None + return [{"score": item["score"] if obj.rule_type == ProblemRuleType.OI else 100, + "input_name": item["input_name"], "output_name": item["output_name"]} + for item in obj.test_case_score] def get_spj(self, obj): return {"code": obj.spj_code, From a52f383930e4828fa915c19456270a950f05f24d Mon Sep 17 00:00:00 2001 From: virusdefender Date: Sat, 5 Jan 2019 14:00:59 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E5=B0=86=E7=BC=96=E8=AF=91=E9=80=89?= =?UTF-8?q?=E9=A1=B9=E4=BF=AE=E6=94=B9=E4=B8=BA=E6=95=B0=E6=8D=AE=E5=BA=93?= =?UTF-8?q?=E5=AD=98=E5=82=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- conf/views.py | 3 +-- judge/dispatcher.py | 7 +++---- judge/languages.py | 6 ------ options/options.py | 26 ++++++++++++++++++++++++ problem/serializers.py | 12 +++++------ problem/views/admin.py | 8 ++++---- submission/serializers.py | 4 ++-- utils/serializers.py | 42 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 84 insertions(+), 24 deletions(-) create mode 100644 utils/serializers.py diff --git a/conf/views.py b/conf/views.py index 875832d9..b0b4cad7 100644 --- a/conf/views.py +++ b/conf/views.py @@ -17,7 +17,6 @@ from account.decorators import super_admin_required from account.models import User from contest.models import Contest from judge.dispatcher import process_pending_task -from judge.languages import languages, spj_languages from options.options import SysOptions from problem.models import Problem from submission.models import Submission @@ -166,7 +165,7 @@ class JudgeServerHeartbeatAPI(CSRFExemptAPIView): class LanguagesAPI(APIView): def get(self, request): - return self.success({"languages": languages, "spj_languages": spj_languages}) + return self.success({"languages": SysOptions.languages, "spj_languages": SysOptions.spj_languages}) class TestCasePruneAPI(APIView): diff --git a/judge/dispatcher.py b/judge/dispatcher.py index c4480956..6f3a9434 100644 --- a/judge/dispatcher.py +++ b/judge/dispatcher.py @@ -10,7 +10,6 @@ from django.db.models import F from account.models import User from conf.models import JudgeServer from contest.models import ContestRuleType, ACMContestRank, OIContestRank, ContestStatus -from judge.languages import languages, spj_languages from options.options import SysOptions from problem.models import Problem, ProblemRuleType from problem.utils import parse_problem_template @@ -66,7 +65,7 @@ class DispatcherBase(object): class SPJCompiler(DispatcherBase): def __init__(self, spj_code, spj_version, spj_language): super().__init__() - spj_compile_config = list(filter(lambda config: spj_language == config["name"], spj_languages))[0]["spj"][ + spj_compile_config = list(filter(lambda config: spj_language == config["name"], SysOptions.spj_languages))[0]["spj"][ "compile"] self.data = { "src": spj_code, @@ -126,10 +125,10 @@ class JudgeDispatcher(DispatcherBase): return language = self.submission.language - sub_config = list(filter(lambda item: language == item["name"], languages))[0] + sub_config = list(filter(lambda item: language == item["name"], SysOptions.languages))[0] spj_config = {} if self.problem.spj_code: - for lang in spj_languages: + for lang in SysOptions.spj_languages: if lang["name"] == self.problem.spj_language: spj_config = lang["spj"] break diff --git a/judge/languages.py b/judge/languages.py index b8d9d36a..68495084 100644 --- a/judge/languages.py +++ b/judge/languages.py @@ -179,9 +179,3 @@ languages = [ {"config": _py2_lang_config, "name": "Python2", "description": "Python 2.7", "content_type": "text/x-python"}, {"config": _py3_lang_config, "name": "Python3", "description": "Python 3.5", "content_type": "text/x-python"}, ] - -spj_languages = list(filter(lambda item: "spj" in item, languages)) - - -language_names = [item["name"] for item in languages] -spj_language_names = [item["name"] for item in spj_languages] diff --git a/options/options.py b/options/options.py index 57169bb7..605c7462 100644 --- a/options/options.py +++ b/options/options.py @@ -4,6 +4,7 @@ from django.db import transaction, IntegrityError from utils.constants import CacheKey from utils.shortcuts import rand_str +from judge.languages import languages from .models import SysOptions as SysOptionsModel @@ -22,6 +23,7 @@ class OptionKeys: smtp_config = "smtp_config" judge_server_token = "judge_server_token" throttling = "throttling" + languages = "languages" class OptionDefaultValue: @@ -35,6 +37,7 @@ class OptionDefaultValue: judge_server_token = default_token throttling = {"ip": {"capacity": 100, "fill_rate": 0.1, "default_capacity": 50}, "user": {"capacity": 20, "fill_rate": 0.03, "default_capacity": 10}} + languages = languages class _SysOptionsMeta(type): @@ -191,6 +194,29 @@ class _SysOptionsMeta(type): def throttling(cls, value): cls._set_option(OptionKeys.throttling, value) + @property + def languages(cls): + return cls._get_option(OptionKeys.languages) + + @languages.setter + def languages(cls, value): + cls._set_option(OptionKeys.languages, value) + + @property + def spj_languages(cls): + return [item for item in cls.languages if "spj" in item] + + @property + def language_names(cls): + return [item["name"] for item in languages] + + @property + def spj_language_names(cls): + return [item["name"] for item in cls.languages if "spj" in item] + + def reset_languages(cls): + cls.languages = languages + class SysOptions(metaclass=_SysOptionsMeta): pass diff --git a/problem/serializers.py b/problem/serializers.py index de9bfdf6..592c758a 100644 --- a/problem/serializers.py +++ b/problem/serializers.py @@ -1,9 +1,9 @@ from django import forms from options.options import SysOptions -from judge.languages import language_names, spj_language_names from utils.api import UsernameSerializer, serializers from utils.constants import Difficulty +from utils.serializers import LanguageNameMultiChoiceField, SPJLanguageNameChoiceField, LanguageNameChoiceField from .models import Problem, ProblemRuleType, ProblemTag from .utils import parse_problem_template @@ -40,11 +40,11 @@ class CreateOrEditProblemSerializer(serializers.Serializer): test_case_score = serializers.ListField(child=CreateTestCaseScoreSerializer(), allow_empty=True) time_limit = serializers.IntegerField(min_value=1, max_value=1000 * 60) memory_limit = serializers.IntegerField(min_value=1, max_value=1024) - languages = serializers.MultipleChoiceField(choices=language_names) + languages = LanguageNameMultiChoiceField() template = serializers.DictField(child=serializers.CharField(min_length=1)) rule_type = serializers.ChoiceField(choices=[ProblemRuleType.ACM, ProblemRuleType.OI]) spj = serializers.BooleanField() - spj_language = serializers.ChoiceField(choices=spj_language_names, allow_blank=True, allow_null=True) + spj_language = SPJLanguageNameChoiceField(allow_blank=True, allow_null=True) spj_code = serializers.CharField(allow_blank=True, allow_null=True) spj_compile_ok = serializers.BooleanField(default=False) visible = serializers.BooleanField() @@ -78,7 +78,7 @@ class TagSerializer(serializers.ModelSerializer): class CompileSPJSerializer(serializers.Serializer): - spj_language = serializers.ChoiceField(choices=spj_language_names) + spj_language = SPJLanguageNameChoiceField() spj_code = serializers.CharField() @@ -212,12 +212,12 @@ class TemplateSerializer(serializers.Serializer): class SPJSerializer(serializers.Serializer): code = serializers.CharField() - language = serializers.ChoiceField(choices=spj_language_names) + language = SPJLanguageNameChoiceField() class AnswerSerializer(serializers.Serializer): code = serializers.CharField() - language = serializers.ChoiceField(choices=language_names) + language = LanguageNameChoiceField() class ImportProblemSerializer(serializers.Serializer): diff --git a/problem/views/admin.py b/problem/views/admin.py index a357cea5..e06ff619 100644 --- a/problem/views/admin.py +++ b/problem/views/admin.py @@ -15,7 +15,7 @@ from account.decorators import problem_permission_required, ensure_created_by from contest.models import Contest, ContestStatus from fps.parser import FPSHelper, FPSParser from judge.dispatcher import SPJCompiler -from judge.languages import language_names +from options.options import SysOptions from submission.models import Submission, JudgeStatus from utils.api import APIView, CSRFExemptAPIView, validate_serializer, APIError from utils.constants import Difficulty @@ -578,7 +578,7 @@ class ImportProblemAPI(CSRFExemptAPIView, TestCaseZipProcessor): else: problem_info = serializer.data for item in problem_info["template"].keys(): - if item not in language_names: + if item not in SysOptions.language_names: return self.error(f"Unsupported language {item}") problem_info["display_id"] = problem_info["display_id"][:24] @@ -613,7 +613,7 @@ class ImportProblemAPI(CSRFExemptAPIView, TestCaseZipProcessor): spj_language=problem_info["spj"][ "language"] if spj else None, spj_version=rand_str(8) if spj else "", - languages=language_names, + languages=SysOptions.language_names, created_by=request.user, visible=False, difficulty=Difficulty.MID, @@ -666,7 +666,7 @@ class FPSProblemImport(CSRFExemptAPIView): spj_language=problem_data["spj"]["language"] if spj else None, spj_version=rand_str(8) if spj else "", visible=False, - languages=language_names, + languages=SysOptions.language_names, created_by=creator, difficulty=Difficulty.MID, test_case_id=problem_data["test_case_id"]) diff --git a/submission/serializers.py b/submission/serializers.py index 3f1e367d..b814bc7a 100644 --- a/submission/serializers.py +++ b/submission/serializers.py @@ -1,11 +1,11 @@ from .models import Submission from utils.api import serializers -from judge.languages import language_names +from utils.serializers import LanguageNameChoiceField class CreateSubmissionSerializer(serializers.Serializer): problem_id = serializers.IntegerField() - language = serializers.ChoiceField(choices=language_names) + language = LanguageNameChoiceField() code = serializers.CharField(max_length=1024 * 1024) contest_id = serializers.IntegerField(required=False) captcha = serializers.CharField(required=False) diff --git a/utils/serializers.py b/utils/serializers.py new file mode 100644 index 00000000..c543c56f --- /dev/null +++ b/utils/serializers.py @@ -0,0 +1,42 @@ +from rest_framework import serializers + +from options.options import SysOptions + + +class InvalidLanguage(serializers.ValidationError): + def __init__(self, name): + super().__init__(detail=f"{name} is not a valid language") + + +class LanguageNameChoiceField(serializers.CharField): + def to_internal_value(self, data): + data = super().to_internal_value(data) + if data and data not in SysOptions.language_names: + raise InvalidLanguage(data) + return data + + +class SPJLanguageNameChoiceField(serializers.CharField): + def to_internal_value(self, data): + data = super().to_internal_value(data) + if data and data not in SysOptions.spj_language_names: + raise InvalidLanguage(data) + return data + + +class LanguageNameMultiChoiceField(serializers.ListField): + def to_internal_value(self, data): + data = super().to_internal_value(data) + for item in data: + if item not in SysOptions.language_names: + raise InvalidLanguage(item) + return data + + +class SPJLanguageNameMultiChoiceField(serializers.ListField): + def to_internal_value(self, data): + data = super().to_internal_value(data) + for item in data: + if item not in SysOptions.spj_language_names: + raise InvalidLanguage(item) + return data From 5ae50e4e54f64c68385270ceb0107ddb1f4285f9 Mon Sep 17 00:00:00 2001 From: virusdefender Date: Sat, 5 Jan 2019 14:53:55 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E8=AE=BE=E7=BD=AE=20gunicorn=20=E5=92=8C?= =?UTF-8?q?=20celery=20=E7=9A=84=20worker=20=E6=95=B0=E9=87=8F=EF=BC=8C?= =?UTF-8?q?=E6=9C=80=E5=B0=8F=E6=98=AF=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy/entrypoint.sh | 9 +++++++++ deploy/supervisord.conf | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/deploy/entrypoint.sh b/deploy/entrypoint.sh index 87ed1e61..627c75a5 100755 --- a/deploy/entrypoint.sh +++ b/deploy/entrypoint.sh @@ -37,6 +37,15 @@ else sed -i "s/__IP_HEADER__/\$remote_addr/g" api_proxy.conf; fi +if [ -z "$MAX_WORKER_NUM" ]; then + export CPU_CORE_NUM=$(grep -c ^processor /proc/cpuinfo) + if [[ $CPU_CORE_NUM -lt 2 ]]; then + export MAX_WORKER_NUM=2 + else + export MAX_WORKER_NUM=$(($CPU_CORE_NUM)) + fi +fi + cd $APP/dist if [ ! -z "$STATIC_CDN_HOST" ]; then find . -name "*.*" -type f -exec sed -i "s/__STATIC_CDN_HOST__/\/$STATIC_CDN_HOST/g" {} \; diff --git a/deploy/supervisord.conf b/deploy/supervisord.conf index 1b946993..0eca7211 100644 --- a/deploy/supervisord.conf +++ b/deploy/supervisord.conf @@ -28,7 +28,7 @@ stopwaitsecs = 5 killasgroup=true [program:gunicorn] -command=sh -c "gunicorn oj.wsgi --user server --group spj -b 127.0.0.1:8080 --reload -w `grep -c ^processor /proc/cpuinfo`" +command=gunicorn oj.wsgi --user server --group spj --bind 127.0.0.1:8080 --workers %(ENV_MAX_WORKER_NUM)s --threads 4 --max-requests-jitter 10000 --max-requests 1000000 --keep-alive 32 directory=/app/ stdout_logfile=/data/log/gunicorn.log stderr_logfile=/data/log/gunicorn.log @@ -39,7 +39,7 @@ stopwaitsecs = 5 killasgroup=true [program:celery] -command=celery -A oj worker -l warning +command=celery -A oj worker -l warning --autoscale 2,%(ENV_MAX_WORKER_NUM)s directory=/app/ user=nobody stdout_logfile=/data/log/celery.log