Merge pull request #212 from QingdaoU/bugfix/fix1

Bugfix/fix1

.gitignore

@@ -76,3 +76,4 @@ data/config/*
 http_locations.conf
 https_locations.conf
 
+venv/

.travis.yml

@@ -2,15 +2,11 @@ language: python
 python:
     - "3.6"
 services:
-    - redis-server
     - docker
-before_install:
-    - docker pull postgres:10
-    - docker run -it -d -e POSTGRES_DB=onlinejudge -e POSTGRES_USER=onlinejudge -e POSTGRES_PASSWORD=onlinejudge -p 127.0.0.1:5433:5432 postgres:10
 install:
     - pip install -r deploy/requirements.txt
     - echo `cat /dev/urandom | head -1 | md5sum | head -c 32` > data/config/secret.key
-    - sleep 10 && python manage.py migrate
+    - ./init_db.sh
 script:
     - docker ps -a
     - flake8 .

conf/views.py

@@ -17,7 +17,6 @@ from account.decorators import super_admin_required
 from account.models import User
 from contest.models import Contest
 from judge.dispatcher import process_pending_task
-from judge.languages import languages, spj_languages
 from options.options import SysOptions
 from problem.models import Problem
 from submission.models import Submission
@@ -166,7 +165,7 @@ class JudgeServerHeartbeatAPI(CSRFExemptAPIView):
 
 class LanguagesAPI(APIView):
     def get(self, request):
-        return self.success({"languages": languages, "spj_languages": spj_languages})
+        return self.success({"languages": SysOptions.languages, "spj_languages": SysOptions.spj_languages})
 
 
 class TestCasePruneAPI(APIView):

deploy/entrypoint.sh

@@ -37,6 +37,15 @@ else
     sed -i "s/__IP_HEADER__/\$remote_addr/g" api_proxy.conf;
 fi
 
+if [ -z "$MAX_WORKER_NUM" ]; then
+    export CPU_CORE_NUM=$(grep -c ^processor /proc/cpuinfo)
+    if [[ $CPU_CORE_NUM -lt 2 ]]; then
+        export MAX_WORKER_NUM=2
+    else
+        export MAX_WORKER_NUM=$(($CPU_CORE_NUM))
+    fi
+fi
+
 cd $APP/dist
 if [ ! -z "$STATIC_CDN_HOST" ]; then
     find . -name "*.*" -type f -exec sed -i "s/__STATIC_CDN_HOST__/\/$STATIC_CDN_HOST/g" {} \;

deploy/supervisord.conf

@@ -28,7 +28,7 @@ stopwaitsecs = 5
 killasgroup=true
 
 [program:gunicorn]
-command=sh -c "gunicorn oj.wsgi --user server --group spj  -b 127.0.0.1:8080 --reload -w `grep -c ^processor /proc/cpuinfo`"
+command=gunicorn oj.wsgi --user server --group spj --bind 127.0.0.1:8080 --workers %(ENV_MAX_WORKER_NUM)s --threads 4 --max-requests-jitter 10000 --max-requests 1000000 --keep-alive 32
 directory=/app/
 stdout_logfile=/data/log/gunicorn.log
 stderr_logfile=/data/log/gunicorn.log
@@ -39,7 +39,7 @@ stopwaitsecs = 5
 killasgroup=true
 
 [program:celery]
-command=celery -A oj worker -l warning
+command=celery -A oj worker -l warning --autoscale 2,%(ENV_MAX_WORKER_NUM)s
 directory=/app/
 user=nobody
 stdout_logfile=/data/log/celery.log

init_db.sh

@@ -0,0 +1,19 @@
+#! /bin/bash
+set -x
+
+if [[ ! -f manage.py ]]; then
+    echo "No manage.py, wrong location"
+    exit 1
+fi
+
+sleep 2
+docker rm -f oj-postgres-dev oj-redis-dev
+docker run -it -d -e POSTGRES_DB=onlinejudge -e POSTGRES_USER=onlinejudge -e POSTGRES_PASSWORD=onlinejudge -p 127.0.0.1:5435:5432 --name oj-postgres-dev postgres:10
+docker run -it -d -p 127.0.0.1:6380:6379 --name oj-redis-dev redis:4.0-alpine
+
+if [ "$1" = "--migrate" ]; then
+    sleep 3
+    echo `cat /dev/urandom | head -1 | md5sum | head -c 32` > data/config/secret.key
+    python manage.py migrate
+    python manage.py inituser --username root --password rootroot --action create_super_admin
+fi

judge/dispatcher.py

@@ -10,7 +10,6 @@ from django.db.models import F
 from account.models import User
 from conf.models import JudgeServer
 from contest.models import ContestRuleType, ACMContestRank, OIContestRank, ContestStatus
-from judge.languages import languages, spj_languages
 from options.options import SysOptions
 from problem.models import Problem, ProblemRuleType
 from problem.utils import parse_problem_template
@@ -66,7 +65,7 @@ class DispatcherBase(object):
 class SPJCompiler(DispatcherBase):
     def __init__(self, spj_code, spj_version, spj_language):
         super().__init__()
-        spj_compile_config = list(filter(lambda config: spj_language == config["name"], spj_languages))[0]["spj"][
+        spj_compile_config = list(filter(lambda config: spj_language == config["name"], SysOptions.spj_languages))[0]["spj"][
             "compile"]
         self.data = {
             "src": spj_code,
@@ -126,10 +125,10 @@ class JudgeDispatcher(DispatcherBase):
             return
 
         language = self.submission.language
-        sub_config = list(filter(lambda item: language == item["name"], languages))[0]
+        sub_config = list(filter(lambda item: language == item["name"], SysOptions.languages))[0]
         spj_config = {}
         if self.problem.spj_code:
-            for lang in spj_languages:
+            for lang in SysOptions.spj_languages:
                 if lang["name"] == self.problem.spj_language:
                     spj_config = lang["spj"]
                     break

judge/languages.py

@@ -179,9 +179,3 @@ languages = [
     {"config": _py2_lang_config, "name": "Python2", "description": "Python 2.7", "content_type": "text/x-python"},
     {"config": _py3_lang_config, "name": "Python3", "description": "Python 3.5", "content_type": "text/x-python"},
 ]
-
-spj_languages = list(filter(lambda item: "spj" in item, languages))
-
-
-language_names = [item["name"] for item in languages]
-spj_language_names = [item["name"] for item in spj_languages]

oj/dev_settings.py

@@ -7,7 +7,7 @@ DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.postgresql_psycopg2',
         'HOST': '127.0.0.1',
-        'PORT': 5433,
+        'PORT': 5435,
         'NAME': "onlinejudge",
         'USER': "onlinejudge",
         'PASSWORD': 'onlinejudge'
@@ -16,7 +16,7 @@ DATABASES = {
 
 REDIS_CONF = {
     "host": "127.0.0.1",
-    "port": "6379"
+    "port": "6380"
 }
 
 

options/options.py

@@ -4,6 +4,7 @@ from django.db import transaction, IntegrityError
 
 from utils.constants import CacheKey
 from utils.shortcuts import rand_str
+from judge.languages import languages
 from .models import SysOptions as SysOptionsModel
 
 
@@ -22,6 +23,7 @@ class OptionKeys:
     smtp_config = "smtp_config"
     judge_server_token = "judge_server_token"
     throttling = "throttling"
+    languages = "languages"
 
 
 class OptionDefaultValue:
@@ -35,6 +37,7 @@ class OptionDefaultValue:
     judge_server_token = default_token
     throttling = {"ip": {"capacity": 100, "fill_rate": 0.1, "default_capacity": 50},
                   "user": {"capacity": 20, "fill_rate": 0.03, "default_capacity": 10}}
+    languages = languages
 
 
 class _SysOptionsMeta(type):
@@ -191,6 +194,29 @@ class _SysOptionsMeta(type):
     def throttling(cls, value):
         cls._set_option(OptionKeys.throttling, value)
 
+    @property
+    def languages(cls):
+        return cls._get_option(OptionKeys.languages)
+
+    @languages.setter
+    def languages(cls, value):
+        cls._set_option(OptionKeys.languages, value)
+
+    @property
+    def spj_languages(cls):
+        return [item for item in cls.languages if "spj" in item]
+
+    @property
+    def language_names(cls):
+        return [item["name"] for item in languages]
+
+    @property
+    def spj_language_names(cls):
+        return [item["name"] for item in cls.languages if "spj" in item]
+
+    def reset_languages(cls):
+        cls.languages = languages
+
 
 class SysOptions(metaclass=_SysOptionsMeta):
     pass

problem/serializers.py

@@ -1,9 +1,9 @@
 from django import forms
 
 from options.options import SysOptions
-from judge.languages import language_names, spj_language_names
 from utils.api import UsernameSerializer, serializers
 from utils.constants import Difficulty
+from utils.serializers import LanguageNameMultiChoiceField, SPJLanguageNameChoiceField, LanguageNameChoiceField
 
 from .models import Problem, ProblemRuleType, ProblemTag
 from .utils import parse_problem_template
@@ -40,11 +40,11 @@ class CreateOrEditProblemSerializer(serializers.Serializer):
     test_case_score = serializers.ListField(child=CreateTestCaseScoreSerializer(), allow_empty=True)
     time_limit = serializers.IntegerField(min_value=1, max_value=1000 * 60)
     memory_limit = serializers.IntegerField(min_value=1, max_value=1024)
-    languages = serializers.MultipleChoiceField(choices=language_names)
+    languages = LanguageNameMultiChoiceField()
     template = serializers.DictField(child=serializers.CharField(min_length=1))
     rule_type = serializers.ChoiceField(choices=[ProblemRuleType.ACM, ProblemRuleType.OI])
     spj = serializers.BooleanField()
-    spj_language = serializers.ChoiceField(choices=spj_language_names, allow_blank=True, allow_null=True)
+    spj_language = SPJLanguageNameChoiceField(allow_blank=True, allow_null=True)
     spj_code = serializers.CharField(allow_blank=True, allow_null=True)
     spj_compile_ok = serializers.BooleanField(default=False)
     visible = serializers.BooleanField()
@@ -78,7 +78,7 @@ class TagSerializer(serializers.ModelSerializer):
 
 
 class CompileSPJSerializer(serializers.Serializer):
-    spj_language = serializers.ChoiceField(choices=spj_language_names)
+    spj_language = SPJLanguageNameChoiceField()
     spj_code = serializers.CharField()
 
 
@@ -154,8 +154,9 @@ class ExportProblemSerializer(serializers.ModelSerializer):
         return self._html_format_value(obj.hint)
 
     def get_test_case_score(self, obj):
-        return [{"score": item["score"], "input_name": item["input_name"], "output_name": item["output_name"]}
-                for item in obj.test_case_score] if obj.rule_type == ProblemRuleType.OI else None
+        return [{"score": item["score"] if obj.rule_type == ProblemRuleType.OI else 100,
+                 "input_name": item["input_name"], "output_name": item["output_name"]}
+                for item in obj.test_case_score]
 
     def get_spj(self, obj):
         return {"code": obj.spj_code,
@@ -211,12 +212,12 @@ class TemplateSerializer(serializers.Serializer):
 
 class SPJSerializer(serializers.Serializer):
     code = serializers.CharField()
-    language = serializers.ChoiceField(choices=spj_language_names)
+    language = SPJLanguageNameChoiceField()
 
 
 class AnswerSerializer(serializers.Serializer):
     code = serializers.CharField()
-    language = serializers.ChoiceField(choices=language_names)
+    language = LanguageNameChoiceField()
 
 
 class ImportProblemSerializer(serializers.Serializer):

problem/views/admin.py

@@ -15,7 +15,7 @@ from account.decorators import problem_permission_required, ensure_created_by
 from contest.models import Contest, ContestStatus
 from fps.parser import FPSHelper, FPSParser
 from judge.dispatcher import SPJCompiler
-from judge.languages import language_names
+from options.options import SysOptions
 from submission.models import Submission, JudgeStatus
 from utils.api import APIView, CSRFExemptAPIView, validate_serializer, APIError
 from utils.constants import Difficulty
@@ -578,7 +578,7 @@ class ImportProblemAPI(CSRFExemptAPIView, TestCaseZipProcessor):
                         else:
                             problem_info = serializer.data
                             for item in problem_info["template"].keys():
-                                if item not in language_names:
+                                if item not in SysOptions.language_names:
                                     return self.error(f"Unsupported language {item}")
 
                         problem_info["display_id"] = problem_info["display_id"][:24]
@@ -613,7 +613,7 @@ class ImportProblemAPI(CSRFExemptAPIView, TestCaseZipProcessor):
                                                              spj_language=problem_info["spj"][
                                                                  "language"] if spj else None,
                                                              spj_version=rand_str(8) if spj else "",
-                                                             languages=language_names,
+                                                             languages=SysOptions.language_names,
                                                              created_by=request.user,
                                                              visible=False,
                                                              difficulty=Difficulty.MID,
@@ -666,7 +666,7 @@ class FPSProblemImport(CSRFExemptAPIView):
                                spj_language=problem_data["spj"]["language"] if spj else None,
                                spj_version=rand_str(8) if spj else "",
                                visible=False,
-                               languages=language_names,
+                               languages=SysOptions.language_names,
                                created_by=creator,
                                difficulty=Difficulty.MID,
                                test_case_id=problem_data["test_case_id"])

submission/serializers.py

@@ -1,11 +1,11 @@
 from .models import Submission
 from utils.api import serializers
-from judge.languages import language_names
+from utils.serializers import LanguageNameChoiceField
 
 
 class CreateSubmissionSerializer(serializers.Serializer):
     problem_id = serializers.IntegerField()
-    language = serializers.ChoiceField(choices=language_names)
+    language = LanguageNameChoiceField()
     code = serializers.CharField(max_length=1024 * 1024)
     contest_id = serializers.IntegerField(required=False)
     captcha = serializers.CharField(required=False)

utils/serializers.py

@@ -0,0 +1,42 @@
+from rest_framework import serializers
+
+from options.options import SysOptions
+
+
+class InvalidLanguage(serializers.ValidationError):
+    def __init__(self, name):
+        super().__init__(detail=f"{name} is not a valid language")
+
+
+class LanguageNameChoiceField(serializers.CharField):
+    def to_internal_value(self, data):
+        data = super().to_internal_value(data)
+        if data and data not in SysOptions.language_names:
+            raise InvalidLanguage(data)
+        return data
+
+
+class SPJLanguageNameChoiceField(serializers.CharField):
+    def to_internal_value(self, data):
+        data = super().to_internal_value(data)
+        if data and data not in SysOptions.spj_language_names:
+            raise InvalidLanguage(data)
+        return data
+
+
+class LanguageNameMultiChoiceField(serializers.ListField):
+    def to_internal_value(self, data):
+        data = super().to_internal_value(data)
+        for item in data:
+            if item not in SysOptions.language_names:
+                raise InvalidLanguage(item)
+        return data
+
+
+class SPJLanguageNameMultiChoiceField(serializers.ListField):
+    def to_internal_value(self, data):
+        data = super().to_internal_value(data)
+        for item in data:
+            if item not in SysOptions.spj_language_names:
+                raise InvalidLanguage(item)
+        return data