From b687d2067b28f6759a47ddcfe7bb575e8f83f679 Mon Sep 17 00:00:00 2001 From: virusdefender <1670873886@qq.com> Date: Sat, 12 Dec 2015 15:41:57 +0800 Subject: [PATCH] =?UTF-8?q?=E5=BA=9F=E5=BC=83=20huey=EF=BC=8C=E5=A4=9A?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BA=93=E8=BF=9E=E6=8E=A5=E7=9A=84=E6=97=B6?= =?UTF-8?q?=E5=80=99=E5=AD=98=E5=9C=A8=20connection=20=E6=97=A0=E6=B3=95?= =?UTF-8?q?=E9=87=8A=E6=94=BE=E7=9A=84=E9=97=AE=E9=A2=98=EF=BC=8C=E5=9B=9E?= =?UTF-8?q?=E5=88=B0=20celery?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 +- account/tasks.py | 8 +++++ account/views.py | 37 ++++++++++++---------- db1.sqlite3 | 0 dockerfiles/oj_web_server/requirements.txt | 3 +- dockerfiles/oj_web_server/task_queue.conf | 4 +-- judge_dispatcher/tasks.py | 4 +-- oj/__init__.py | 5 +++ oj/celery.py | 19 +++++++++++ oj/local_settings.py | 6 ++++ oj/server_settings.py | 6 ++++ oj/settings.py | 15 +++------ submission/tasks.py | 10 +++--- submission/views.py | 6 ++-- 14 files changed, 85 insertions(+), 40 deletions(-) create mode 100644 account/tasks.py delete mode 100644 db1.sqlite3 create mode 100644 oj/celery.py diff --git a/.gitignore b/.gitignore index e0873616..8707498f 100644 --- a/.gitignore +++ b/.gitignore @@ -52,7 +52,7 @@ db.db #redis dump *.rdb #*.out -db.sqlite3 +*.sqlite3 .DS_Store log/ static/release/css diff --git a/account/tasks.py b/account/tasks.py new file mode 100644 index 00000000..58cb8fc1 --- /dev/null +++ b/account/tasks.py @@ -0,0 +1,8 @@ +# coding=utf-8 +from celery import shared_task +from utils.mail import send_email + + +@shared_task +def _send_email(from_name, to_email, to_name, subject, content): + send_email(from_name, to_email, to_name, subject, content) \ No newline at end of file diff --git a/account/views.py b/account/views.py index b02c880a..502e0363 100644 --- a/account/views.py +++ b/account/views.py @@ -14,7 +14,7 @@ from rest_framework.response import Response from utils.shortcuts import (serializer_invalid_response, error_response, success_response, error_page, paginate, rand_str) from utils.captcha import Captcha -from utils.mail import send_email +from .tasks import _send_email from .decorators import login_required from .models import User, UserProfile @@ -63,7 +63,7 @@ def index_page(request): return render(request, "oj/index.html") if request.META.get('HTTP_REFERER') or request.GET.get("index"): - return render(request, "oj/index.html") + return render(request, "oj/index.html") else: return http.HttpResponseRedirect('/problems/') @@ -151,9 +151,9 @@ class EmailCheckAPIView(APIView): 检测邮箱是否存在，用状态码标识结果 --- """ - #这里是为了适应前端表单验证空间的要求 + # 这里是为了适应前端表单验证空间的要求 reset = request.GET.get("reset", None) - #如果reset为true说明该请求是重置密码页面发出的，要返回的状态码应正好相反 + # 如果reset为true说明该请求是重置密码页面发出的，要返回的状态码应正好相反 if reset: existed = 200 does_not_existed = 400 @@ -287,22 +287,25 @@ class ApplyResetPasswordAPIView(APIView): user = User.objects.get(email=data["email"]) except User.DoesNotExist: return error_response(u"用户不存在") - if user.reset_password_token_create_time and (now() - user.reset_password_token_create_time).total_seconds() < 20 * 60: + if user.reset_password_token_create_time and ( + now() - user.reset_password_token_create_time).total_seconds() < 20 * 60: return error_response(u"20分钟内只能找回一次密码") user.reset_password_token = rand_str() user.reset_password_token_create_time = now() user.save() - email_template = codecs.open(settings.TEMPLATES[0]["DIRS"][0] + "utils/reset_password_email.html", "r", "utf-8").read() + email_template = codecs.open(settings.TEMPLATES[0]["DIRS"][0] + "utils/reset_password_email.html", "r", + "utf-8").read() - email_template = email_template.replace("{{ username }}", user.username).\ - replace("{{ website_name }}", settings.WEBSITE_INFO["website_name"]).\ - replace("{{ link }}", request.scheme + "://" + request.META['HTTP_HOST'] + "/reset_password/t/" + user.reset_password_token) + email_template = email_template.replace("{{ username }}", user.username). \ + replace("{{ website_name }}", settings.WEBSITE_INFO["website_name"]). \ + replace("{{ link }}", request.scheme + "://" + request.META[ + 'HTTP_HOST'] + "/reset_password/t/" + user.reset_password_token) - send_email(settings.WEBSITE_INFO["website_name"], - user.email, - user.username, - settings.WEBSITE_INFO["website_name"] + u" 登录信息找回邮件", - email_template) + _send_email.delay(settings.WEBSITE_INFO["website_name"], + user.email, + user.username, + settings.WEBSITE_INFO["website_name"] + u" 登录信息找回邮件", + email_template) return success_response(u"邮件发送成功,请前往您的邮箱查收") else: return serializer_invalid_response(serializer) @@ -352,7 +355,8 @@ class SSOAPIView(APIView): user = User.objects.get(auth_token=serializer.data["token"]) user.auth_token = None user.save() - return success_response({"username": user.username, "admin_type": user.admin_type, "avatar": user.userprofile.avatar}) + return success_response( + {"username": user.username, "admin_type": user.admin_type, "avatar": user.userprofile.avatar}) except User.DoesNotExist: return error_response(u"用户不存在") else: @@ -366,7 +370,8 @@ class SSOAPIView(APIView): token = rand_str() request.user.auth_token = token request.user.save() - return render(request, "oj/account/sso.html", {"redirect_url": callback + "?token=" + token, "callback": callback}) + return render(request, "oj/account/sso.html", + {"redirect_url": callback + "?token=" + token, "callback": callback}) def reset_password_page(request, token): diff --git a/db1.sqlite3 b/db1.sqlite3 deleted file mode 100644 index e69de29b..00000000 diff --git a/dockerfiles/oj_web_server/requirements.txt b/dockerfiles/oj_web_server/requirements.txt index 56676a92..02d03d5c 100644 --- a/dockerfiles/oj_web_server/requirements.txt +++ b/dockerfiles/oj_web_server/requirements.txt @@ -11,4 +11,5 @@ supervisor pillow jsonfield Envelopes -huey \ No newline at end of file +celery +djcelery \ No newline at end of file diff --git a/dockerfiles/oj_web_server/task_queue.conf b/dockerfiles/oj_web_server/task_queue.conf index 39f837e0..5a9b13f7 100644 --- a/dockerfiles/oj_web_server/task_queue.conf +++ b/dockerfiles/oj_web_server/task_queue.conf @@ -1,6 +1,6 @@ -[program:mq] +[program:task_queue] -command=python manage.py run_huey +command=python manage.py celeryd -B -l DEBUG directory=/code/ user=root diff --git a/judge_dispatcher/tasks.py b/judge_dispatcher/tasks.py index 244c9074..4f1b366a 100644 --- a/judge_dispatcher/tasks.py +++ b/judge_dispatcher/tasks.py @@ -32,7 +32,7 @@ class JudgeDispatcher(object): if servers.exists(): return servers.first() - def judge(self, is_waiting_task=False): + def judge(self): self.submission.judge_start_time = int(time.time() * 1000) with transaction.atomic(): @@ -89,7 +89,7 @@ class JudgeDispatcher(object): submission = Submission.objects.get(id=waiting_submission.submission_id) waiting_submission.delete() - _judge(submission, time_limit=waiting_submission.time_limit, + _judge.delay(submission, time_limit=waiting_submission.time_limit, memory_limit=waiting_submission.memory_limit, test_case_id=waiting_submission.test_case_id, is_waiting_task=True) diff --git a/oj/__init__.py b/oj/__init__.py index f4206374..44aca8db 100644 --- a/oj/__init__.py +++ b/oj/__init__.py @@ -7,3 +7,8 @@ |___/ |___/ |_| https://github.com/QingdaoU/OnlineJudge """ +from __future__ import absolute_import + +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app \ No newline at end of file diff --git a/oj/celery.py b/oj/celery.py new file mode 100644 index 00000000..f20f9dc8 --- /dev/null +++ b/oj/celery.py @@ -0,0 +1,19 @@ +from __future__ import absolute_import + +import os + +from celery import Celery + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'oj.settings') + +from django.conf import settings + +app = Celery('oj') + +# Using a string here means the worker will not have to +# pickle the object when using Windows. +app.config_from_object('django.conf:settings') + +# load task modules from all registered Django app configs. +app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) \ No newline at end of file diff --git a/oj/local_settings.py b/oj/local_settings.py index 794a7b63..8fa3b8ac 100644 --- a/oj/local_settings.py +++ b/oj/local_settings.py @@ -28,6 +28,12 @@ REDIS_QUEUE = { "db": 2 } + +# for celery +BROKER_URL = 'redis://%s:%s/%s' % (REDIS_QUEUE["host"], str(REDIS_QUEUE["port"]), str(REDIS_QUEUE["db"])) +ACCEPT_CONTENT = ['json'] + + DEBUG = True ALLOWED_HOSTS = [] diff --git a/oj/server_settings.py b/oj/server_settings.py index afec6d19..98eae980 100644 --- a/oj/server_settings.py +++ b/oj/server_settings.py @@ -37,6 +37,12 @@ REDIS_QUEUE = { "db": 2 } + +# for celery +BROKER_URL = 'redis://%s:%s/%s' % (REDIS_QUEUE["host"], str(REDIS_QUEUE["port"]), str(REDIS_QUEUE["db"])) +ACCEPT_CONTENT = ['json'] + + DEBUG = False ALLOWED_HOSTS = ['*'] diff --git a/oj/settings.py b/oj/settings.py index f5857033..97424d0d 100644 --- a/oj/settings.py +++ b/oj/settings.py @@ -10,7 +10,7 @@ https://docs.djangoproject.com/en/1.8/topics/settings/ For the full list of settings and their values, see https://docs.djangoproject.com/en/1.8/ref/settings/ """ - +from __future__ import absolute_import # Build paths inside the project like this: os.path.join(BASE_DIR, ...) import os @@ -22,6 +22,9 @@ if ENV == "local": elif ENV == "server": from .server_settings import * +import djcelery +djcelery.setup_loader() + BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -53,7 +56,7 @@ INSTALLED_APPS = ( 'judge_dispatcher', 'rest_framework', - 'huey.djhuey', + 'djcelery', ) if DEBUG: @@ -186,14 +189,6 @@ WEBSITE_INFO = {"website_name": "qduoj", "website_footer": u"青岛大学信息工程学院 创新实验室 京ICP备15062075号-1", "url": "https://qduoj.com"} -HUEY = { - 'backend': 'huey.backends.redis_backend', - 'name': 'task_queue', - 'connection': {'host': REDIS_QUEUE["host"], 'port': REDIS_QUEUE["port"], 'db': REDIS_QUEUE["db"]}, - 'always_eager': False, # Defaults to False when running via manage.py run_huey - # Options to pass into the consumer when running ``manage.py run_huey`` - 'consumer_options': {'workers': 50}, -} SMTP_CONFIG = {"smtp_server": "smtp.mxhichina.com", "email": "noreply@qduoj.com", diff --git a/submission/tasks.py b/submission/tasks.py index 0e958e7b..4267841a 100644 --- a/submission/tasks.py +++ b/submission/tasks.py @@ -1,9 +1,9 @@ # coding=utf-8 -from huey.djhuey import db_task - +from __future__ import absolute_import +from celery import shared_task from judge_dispatcher.tasks import JudgeDispatcher -@db_task() -def _judge(submission, time_limit, memory_limit, test_case_id, is_waiting_task=False): - JudgeDispatcher(submission, time_limit, memory_limit, test_case_id).judge(is_waiting_task) \ No newline at end of file +@shared_task +def _judge(submission, time_limit, memory_limit, test_case_id): + JudgeDispatcher(submission, time_limit, memory_limit, test_case_id).judge() \ No newline at end of file diff --git a/submission/views.py b/submission/views.py index ccda8acb..41f5e3b5 100644 --- a/submission/views.py +++ b/submission/views.py @@ -43,7 +43,7 @@ class SubmissionAPIView(APIView): problem_id=problem.id) try: - _judge(submission, problem.time_limit, problem.memory_limit, problem.test_case_id) + _judge.delay(submission, problem.time_limit, problem.memory_limit, problem.test_case_id) except Exception as e: logger.error(e) return error_response(u"提交判题任务失败") @@ -88,7 +88,7 @@ class ContestSubmissionAPIView(APIView): code=data["code"], problem_id=problem.id) try: - _judge(submission, problem.time_limit, problem.memory_limit, problem.test_case_id) + _judge.delay(submission, problem.time_limit, problem.memory_limit, problem.test_case_id) except Exception as e: logger.error(e) return error_response(u"提交判题任务失败") @@ -273,7 +273,7 @@ class SubmissionRejudgeAdminAPIView(APIView): except Problem.DoesNotExist: return error_response(u"题目不存在") try: - _judge(submission, problem.time_limit, problem.memory_limit, problem.test_case_id) + _judge.delay(submission, problem.time_limit, problem.memory_limit, problem.test_case_id) except Exception as e: logger.error(e) return error_response(u"提交判题任务失败")