File upload (#194)

* add form, view, url route

* couple tests for file upload

* file upload coffee, editor update

* coffee-spec jasmine tests

* rename ST_ALLOWED_UPLOAD_FILE_FORMAT, pass to contexts

* import magic, update validations and file upload tests

* allowedFileMedia array as passable option to EditorFileUpload

* cleanup; simple_tag for file media types, update magic validation

* add logging, change ValidationError text
This commit is contained in:
Rick Miyamoto 2017-09-21 16:35:51 +09:00 committed by Esteban Castro Borsani
parent 12f737f410
commit ca4b5e047f
14 changed files with 439 additions and 9 deletions

View File

@ -7,3 +7,4 @@ django-infinite-scroll-pagination>=0.2.0,<0.3
django-djconfig>=0.5.1,<0.6
uni-slugify==0.1.4
pytz
python-magic==0.4.13

View File

@ -4,6 +4,8 @@ from __future__ import unicode_literals
import os
import magic
import logging
from django import forms
from django.conf import settings
from django.core.files.storage import default_storage
@ -16,6 +18,7 @@ from ..topic.models import Topic
from .poll.models import CommentPoll, CommentPollChoice
from .models import Comment
logger = logging.getLogger(__name__)
class CommentForm(forms.ModelForm):
comment = forms.CharField(
@ -138,3 +141,48 @@ class CommentImageForm(forms.Form):
name = default_storage.save(name, file)
file.url = default_storage.url(name)
return file
class CommentFileForm(forms.Form):
file = forms.FileField()
def __init__(self, user=None, *args, **kwargs):
super(CommentFileForm, self).__init__(*args, **kwargs)
self.user = user
def clean_file(self):
file = self.cleaned_data['file']
try:
file_mime = magic.from_buffer(file.read(131072), mime=True)
except magic.MagicException as e:
logger.exception(e)
raise forms.ValidationError(_("The file could not be validated"))
else:
# Won't ever raise. Has at most one '.' so lstrip is fine here
ext = os.path.splitext(file.name)[1].lstrip('.')
mime = settings.ST_ALLOWED_UPLOAD_FILE_MEDIA_TYPE.get(ext, None)
if mime is None:
raise forms.ValidationError(
_("Unsupported file extension %s. Supported extensions are %s."
% (ext, ", ".join(settings.ST_ALLOWED_UPLOAD_FILE_MEDIA_TYPE.keys()))
)
)
if mime != file_mime:
raise forms.ValidationError(
_("Unsupported file mime type %s. Supported types are %s."
% (file_mime, ", ".join(settings.ST_ALLOWED_UPLOAD_FILE_MEDIA_TYPE.values()))
)
)
return file
def save(self):
file = self.cleaned_data['file']
file_hash = utils.get_file_hash(file)
file.name = ''.join((file_hash, '.', file.name.lower()))
name = os.path.join('spirit', 'files', str(self.user.pk), file.name)
name = default_storage.save(name, file)
file.url = default_storage.url(name)
return file

View File

@ -4,6 +4,7 @@ from __future__ import unicode_literals
from django.utils.translation import ugettext as _
from django.utils.html import mark_safe
from django.conf import settings
from ..core.tags.registry import register
from .poll.utils.render import render_polls
@ -14,7 +15,16 @@ from .models import MOVED, CLOSED, UNCLOSED, PINNED, UNPINNED
@register.inclusion_tag('spirit/comment/_form.html')
def render_comments_form(topic, next=None):
form = CommentForm()
return {'form': form, 'topic_id': topic.pk, 'next': next}
return {
'form': form,
'topic_id': topic.pk,
'next': next,
}
@register.simple_tag()
def get_allowed_file_types():
return ".{}".format(", .".join(settings.ST_ALLOWED_UPLOAD_FILE_MEDIA_TYPE.keys()))
@register.simple_tag()

View File

@ -1,4 +1,4 @@
{% load i18n %}
{% load spirit_tags i18n %}
{% load static from staticfiles %}
<div class="comment-text js-box-preview-content" style="display:none;"></div>
@ -8,6 +8,7 @@
--><li><a class="js-box-list" href="#" title="{% trans "List" %}"><i class="fa fa-list"></i></a></li><!--
--><li><a class="js-box-url" href="#" title="{% trans "URL" %}"><i class="fa fa-link"></i></a></li><!--
--><li><a class="js-box-image" href="#" title="{% trans "Image" %}"><i class="fa fa-picture-o"></i></a></li><!--
--><li><a class="js-box-file" href="#" title="{% trans "File" %}"><i class="fa fa-file"></i></a></li><!--
--><li><a class="js-box-poll" href="#" title="{% trans "Poll" %}"><i class="fa fa-bar-chart-o"></i></a></li><!--
--><li><a class="js-box-preview" href="#" title="{% trans "Preview" %}"><i class="fa fa-eye"></i></a></li>
</ul>
@ -26,11 +27,18 @@
smartypants: false
} );
$( '.js-reply' ).find( 'textarea' ).editor_image_upload( {
$( '.js-reply' ).find( 'textarea' )
.editor_image_upload( {
csrfToken: "{{ csrf_token }}",
target: "{% url "spirit:comment:image-upload-ajax" %}",
placeholderText: "{% trans "uploading {image_name}" %}"
} )
.editor_file_upload({
csrfToken: "{{ csrf_token }}",
target: "{% url "spirit:comment:file-upload-ajax" %}",
placeholderText: "{% trans "uploading {file_name}" %}",
allowedFileMedia: "{% get_allowed_file_types %}"
} )
.editor( {
boldedText: "{% trans "bolded text" %}",
italicisedText: "{% trans "italicised text" %}",
@ -39,6 +47,8 @@
linkUrlText: "{% trans "link url" %}",
imageText: "{% trans "image text" %}",
imageUrlText: "{% trans "image url" %}",
fileText: "{% trans "file text" %}",
fileUrlText: "{% trans "file url" %}",
pollTitleText: "{% trans "Title" %}",
pollChoiceText: "{% trans "Description" %}"
} )

View File

@ -456,6 +456,78 @@ class CommentViewTest(TestCase):
self.assertIn('error', res.keys())
self.assertIn('image', res['error'].keys())
@override_settings(MEDIA_ROOT=os.path.join(settings.BASE_DIR, 'media_test'))
def test_comment_file_upload(self):
"""
comment file upload
"""
utils.login(self)
# sample valid pdf - https://stackoverflow.com/a/17280876
file = BytesIO(b'%PDF-1.0\n1 0 obj<</Type/Catalog/Pages 2 0 R>>endobj 2 0 obj<</Type/Pages/Kids[3 0 R]/Count 1'
b'>>endobj 3 0 obj<</Type/Page/MediaBox[0 0 3 3]>>endobj\nxref\n0 4\n0000000000 65535 f\n000000'
b'0010 00000 n\n0000000053 00000 n\n0000000102 00000 n\ntrailer<</Size 4/Root 1 0 R>>\nstartxre'
b'f\n149\n%EOF\n')
files = {'file': SimpleUploadedFile('file.pdf', file.read(), content_type='application/pdf'), }
response = self.client.post(reverse('spirit:comment:file-upload-ajax'),
HTTP_X_REQUESTED_WITH='XMLHttpRequest',
data=files)
res = json.loads(response.content.decode('utf-8'))
file_url = os.path.join(
settings.MEDIA_URL, 'spirit', 'files', str(self.user.pk), "fadcb2389bb2b69b46bc54185de0ae91.file.pdf"
).replace("\\", "/")
self.assertEqual(res['url'], file_url)
file_path = os.path.join(
settings.MEDIA_ROOT, 'spirit', 'files', str(self.user.pk), "fadcb2389bb2b69b46bc54185de0ae91.file.pdf"
)
with open(file_path, 'rb') as fh:
file.seek(0)
self.assertEqual(fh.read(), file.read())
shutil.rmtree(settings.MEDIA_ROOT) # cleanup
def test_comment_file_upload_invalid_ext(self):
"""
comment file upload, invalid file extension
"""
utils.login(self)
# sample valid pdf - https://stackoverflow.com/a/17280876
file = BytesIO(b'%PDF-1.0\n1 0 obj<</Type/Catalog/Pages 2 0 R>>endobj 2 0 obj<</Type/Pages/Kids[3 0 R]/Count 1'
b'>>endobj 3 0 obj<</Type/Page/MediaBox[0 0 3 3]>>endobj\nxref\n0 4\n0000000000 65535 f\n000000'
b'0010 00000 n\n0000000053 00000 n\n0000000102 00000 n\ntrailer<</Size 4/Root 1 0 R>>\nstartxre'
b'f\n149\n%EOF\n')
files = {'file': SimpleUploadedFile('fake.gif', file.read(), content_type='application/pdf'), }
response = self.client.post(reverse('spirit:comment:file-upload-ajax'),
HTTP_X_REQUESTED_WITH='XMLHttpRequest',
data=files)
res = json.loads(response.content.decode('utf-8'))
self.assertIn('error', res.keys())
self.assertIn('file', res['error'].keys())
self.assertEqual(res['error']['file'],
['Unsupported file extension gif. Supported extensions are doc, docx, pdf.'])
def test_comment_file_upload_invalid_mime(self):
"""
comment file upload, invalid mime type
"""
utils.login(self)
file = BytesIO(b'BAD\x02D\x01\x00;')
files = {'file': SimpleUploadedFile('file.pdf', file.read(), content_type='application/pdf'), }
response = self.client.post(reverse('spirit:comment:file-upload-ajax'),
HTTP_X_REQUESTED_WITH='XMLHttpRequest',
data=files)
res = json.loads(response.content.decode('utf-8'))
self.assertIn('error', res.keys())
self.assertIn('file', res['error'].keys())
self.assertEqual(res['error']['file'],
[
'Unsupported file mime type application/octet-stream. '
'Supported types are application/msword, '
'application/vnd.openxmlformats-officedocument.wordprocessingml.document, '
'application/pdf.'])
class CommentModelsTest(TestCase):

View File

@ -24,6 +24,7 @@ urlpatterns = [
url(r'^(?P<pk>\d+)/undelete/$', views.delete, kwargs={'remove': False, }, name='undelete'),
url(r'^upload/$', views.image_upload_ajax, name='image-upload-ajax'),
url(r'^upload/file/$', views.file_upload_ajax, name='file-upload-ajax'),
url(r'^bookmark/', include(spirit.comment.bookmark.urls, namespace='bookmark')),
url(r'^flag/', include(spirit.comment.flag.urls, namespace='flag')),

View File

@ -2,6 +2,7 @@
from __future__ import unicode_literals
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.shortcuts import render, redirect, get_object_or_404
from django.views.decorators.http import require_POST
@ -15,7 +16,7 @@ from ..core.utils.decorators import moderator_required
from ..core.utils import markdown, paginator, render_form_errors, json_response
from ..topic.models import Topic
from .models import Comment
from .forms import CommentForm, CommentMoveForm, CommentImageForm
from .forms import CommentForm, CommentMoveForm, CommentImageForm, CommentFileForm
from .utils import comment_posted, post_comment_update, pre_comment_update
@ -53,7 +54,8 @@ def publish(request, topic_id, pk=None):
context = {
'form': form,
'topic': topic}
'topic': topic,
}
return render(request, 'spirit/comment/publish.html', context)
@ -73,7 +75,9 @@ def update(request, pk):
else:
form = CommentForm(instance=comment)
context = {'form': form, }
context = {
'form': form,
}
return render(request, 'spirit/comment/update.html', context)
@ -135,3 +139,18 @@ def image_upload_ajax(request):
return json_response({'url': image.url, })
return json_response({'error': dict(form.errors.items()), })
@require_POST
@login_required
def file_upload_ajax(request):
if not request.is_ajax():
return Http404()
form = CommentFileForm(user=request.user, data=request.POST, files=request.FILES)
if form.is_valid():
file = form.save()
return json_response({'url': file.url, })
return json_response({'error': dict(form.errors.items()), })

View File

@ -16,6 +16,8 @@ class Editor
linkUrlText: "link url",
imageText: "image text",
imageUrlText: "image url",
fileText: "file text",
fileUrlText: "file url",
pollTitleText: "Title",
pollChoiceText: "Description"
}
@ -35,6 +37,7 @@ class Editor
$('.js-box-list').on('click', @addList)
$('.js-box-url').on('click', @addUrl)
$('.js-box-image').on('click', @addImage)
$('.js-box-file').on('click', @addFile)
$('.js-box-poll').on('click', @addPoll)
$('.js-box-preview').on('click', @togglePreview)
@ -74,6 +77,10 @@ class Editor
@wrapSelection("![", "](#{ @options.imageUrlText })", @options.imageText)
return false
addFile: =>
@wrapSelection("[", "](#{ @options.fileUrlText })", @options.fileText)
return false
addPoll: =>
poll = "\n\n[poll name=#{@pollCounter}]\n" +
"# #{@options.pollTitleText}\n" +

View File

@ -0,0 +1,118 @@
###
Markdown editor image upload, should be loaded before $.editor()
requires: util.js
###
$ = jQuery
class EditorFileUpload
defaults: {
csrfToken: "csrf_token",
target: "target url",
placeholderText: "uploading {file_name}",
allowedFileMedia: [".doc", ".docx", ".pdf"]
}
constructor: (el, options) ->
@el = $(el)
@options = $.extend({}, @defaults, options)
@formFile = $("<form/>")
@inputFile = $("<input/>", {
type: "file",
accept: @options.allowedFileMedia}).appendTo(@formFile)
@setUp()
setUp: ->
if not window.FormData?
return
@inputFile.on('change', @sendFile)
# TODO: fixme, having multiple editors
# in the same page would open several
# dialogs on box-image click
$boxImage = $(".js-box-file")
$boxImage.on('click', @openFileDialog)
$boxImage.on('click', @stopClick)
sendFile: =>
file = @inputFile.get(0).files[0]
placeholder = @addPlaceholder(file)
formData = @buildFormData(file)
post = $.ajax({
url: @options.target,
data: formData,
processData: false,
contentType: false,
type: 'POST'
})
post.done((data) =>
if "url" of data
@addFile(data, file, placeholder)
else
@addError(data, placeholder)
)
post.fail((jqxhr, textStatus, error) =>
@addStatusError(textStatus, error, placeholder)
)
post.always(() =>
# Reset the input after uploading,
# fixes uploading the same image twice
@formFile.get(0).reset()
)
return
addPlaceholder: (file) =>
placeholder = $.format("[#{ @options.placeholderText }]()", {file_name: file.name, })
@el.val(@el.val() + placeholder)
return placeholder
buildFormData: (file) =>
formData = new FormData()
formData.append('csrfmiddlewaretoken', @options.csrfToken)
formData.append('file', file)
return formData
addFile: (data, file, placeholder) =>
# format as a link to the file
fileTag = $.format("[{name}]({url})", {name: file.name, url: data.url})
@textReplace(placeholder, fileTag)
addError: (data, placeholder) =>
error = JSON.stringify(data)
@textReplace(placeholder, "[#{ error }]()")
addStatusError: (textStatus, error, placeholder) =>
errorTag = $.format("[error: {code} {error}]()", {code: textStatus, error: error})
@textReplace(placeholder, errorTag)
textReplace: (find, replace) =>
@el.val(@el.val().replace(find, replace))
return
openFileDialog: =>
@inputFile.trigger('click')
return
stopClick: (e) ->
e.preventDefault()
e.stopPropagation()
e.stopImmediatePropagation()
return
$.fn.extend
editor_file_upload: (options) ->
@each( ->
if not $(@).data('plugin_editor_file_upload')
$(@).data('plugin_editor_file_upload', new EditorFileUpload(@, options))
)
$.fn.editor_file_upload.EditorFileUpload = EditorFileUpload

View File

@ -1,6 +1,7 @@
<form class="js-reply" action=".">
<textarea id="id_comment"></textarea>
<textarea id="id_comment2"></textarea>
<textarea id="id_comment3"></textarea>
<div class="js-box-preview-content" style="display:none;"></div>
@ -10,6 +11,7 @@
<li><a class="js-box-list" href="#" title="List"></a></li>
<li><a class="js-box-url" href="#" title="URL"></a></li>
<li><a class="js-box-image" href="#" title="Image"></a></li>
<li><a class="js-box-file" href="#" title="File"></a></li>
<li><a class="js-box-poll" href="#" title="Poll"></a></li>
<li><a class="js-box-preview" href="#" title="Preview"></a></li>
</ul>

View File

@ -14,7 +14,9 @@ describe "editor plugin tests", ->
linkText: "foo link text",
linkUrlText: "foo link url",
imageText: "foo image text",
imageUrlText: "foo image url"
imageUrlText: "foo image url",
fileText: "foo file text",
fileUrlText: "foo file url"
}
editor = textarea.data 'plugin_editor'
@ -51,12 +53,17 @@ describe "editor plugin tests", ->
$('.js-box-image').trigger 'click'
expect(textarea.val()).toEqual "![foo image text](foo image url)"
it "adds file", ->
$('.js-box-file').trigger 'click'
expect(textarea.val()).toEqual "[foo file text](foo file url)"
it "adds all", ->
$('.js-box-bold').trigger 'click'
$('.js-box-italic').trigger 'click'
$('.js-box-list').trigger 'click'
$('.js-box-url').trigger 'click'
$('.js-box-image').trigger 'click'
$('.js-box-file').trigger 'click'
# expect(textarea.val()).toEqual "![foo image text](foo image url)[foo link text](foo link url)\n* foo list item*foo italicised text***foo bolded text**"
it "wraps the selected text, bold", ->
@ -99,6 +106,14 @@ describe "editor plugin tests", ->
$('.js-box-image').trigger 'click'
expect(textarea.val()).toEqual "bir![foo](foo image url)bar"
it "wraps the selected text, file", ->
textarea.val "birfoobar"
textarea.first()[0].selectionStart = 3
textarea.first()[0].selectionEnd = 6
$('.js-box-file').trigger 'click'
expect(textarea.val()).toEqual "bir[foo](foo file url)bar"
it "shows html preview", ->
textarea.val "*foo*"
$('.js-box-preview').trigger 'click'

View File

@ -0,0 +1,113 @@
describe "editor file upload plugin tests", ->
textarea = null
editorFileUpload = null
data = null
inputFile = null
file = null
post = null
beforeEach ->
fixtures = do jasmine.getFixtures
fixtures.fixturesPath = 'base/test/fixtures/'
loadFixtures 'editor.html'
post = spyOn $, 'ajax'
post.and.callFake (req) ->
d = $.Deferred()
d.resolve(data) # success
#d.reject() # failure
return d.promise()
data =
url: "/path/file.pdf"
file =
name: "foo.pdf"
textarea = $('#id_comment').editor_file_upload {
csrfToken: "foo csrf_token",
target: "/foo/",
placeholderText: "foo uploading {file_name}"
}
editorFileUpload = textarea.first().data 'plugin_editor_file_upload'
inputFile = editorFileUpload.inputFile
it "doesnt break selector chaining", ->
expect(textarea).toEqual $('#id_comment')
expect(textarea.length).toEqual 1
it "does nothing if the browser is not supported", ->
org_formData = window.FormData
window.FormData = null
try
# remove event from beforeEach editor to prevent popup
$(".js-box-file").off 'click'
textarea2 = $('#id_comment2').editor_file_upload()
inputFile2 = textarea2.data('plugin_editor_file_upload').inputFile
trigger = spyOn inputFile2, 'trigger'
$(".js-box-file").trigger 'click'
expect(trigger).not.toHaveBeenCalled()
finally
window.FormData = org_formData
it "opens the file choose dialog", ->
trigger = spyOn inputFile, 'trigger'
$(".js-box-file").trigger 'click'
expect(trigger).toHaveBeenCalled()
it "uploads the file", ->
expect($.ajax.calls.any()).toEqual false
formDataMock = jasmine.createSpyObj('formDataMock', ['append', ])
spyOn(window, "FormData").and.returnValue formDataMock
spyOn(inputFile, 'get').and.returnValue {files: [file, ]}
inputFile.trigger 'change'
expect($.ajax.calls.any()).toEqual true
expect($.ajax.calls.argsFor(0)).toEqual [ { url: '/foo/', data: formDataMock, processData: false, contentType: false, type: 'POST' } ]
expect(formDataMock.append).toHaveBeenCalledWith('csrfmiddlewaretoken', 'foo csrf_token')
expect(formDataMock.append).toHaveBeenCalledWith('file', { name : 'foo.pdf' })
it "changes the placeholder on upload success", ->
textarea.val "foobar"
spyOn(inputFile, 'get').and.returnValue {files: [file, ]}
inputFile.trigger 'change'
expect(textarea.val()).toEqual "foobar[foo.pdf](/path/file.pdf)"
it "changes the placeholder on upload error", ->
textarea.val "foobar"
data =
error: {foo: "foo error", }
spyOn(inputFile, 'get').and.returnValue {files: [file, ]}
inputFile.trigger 'change'
expect(textarea.val()).toEqual "foobar[{\"error\":{\"foo\":\"foo error\"}}]()"
it "changes the placeholder on upload failure", ->
textarea.val "foobar"
d = $.Deferred()
post.and.callFake (req) ->
d.reject(null, "foo statusError", "bar error") # failure
return d.promise()
spyOn(inputFile, 'get').and.returnValue {files: [file, ]}
inputFile.trigger 'change'
expect(textarea.val()).toEqual "foobar[error: foo statusError bar error]()"
it "checks for default media file extensions if none are provided", ->
expect(inputFile[0].outerHTML).toContain(".doc,.docx,.pdf")
it "checks for custom media file extensions if they are provided", ->
textarea3 = $('#id_comment3').editor_file_upload {
csrfToken: "foo csrf_token",
target: "/foo/",
placeholderText: "foo uploading {file_name}"
allowedFileMedia: [".superdoc"]
}
editorFileUpload3 = textarea3.first().data 'plugin_editor_file_upload'
inputFile3 = editorFileUpload3.inputFile
expect(inputFile3[0].outerHTML).not.toContain(".doc,.docx,.pdf")
expect(inputFile3[0].outerHTML).toContain(".superdoc")

View File

@ -5,6 +5,7 @@
from __future__ import unicode_literals
import os
from collections import OrderedDict
ST_TOPIC_PRIVATE_CATEGORY_PK = 1
@ -33,6 +34,15 @@ ST_PRIVATE_FORUM = False
# followed by malicious HTML. See:
# https://docs.djangoproject.com/en/1.11/topics/security/#user-uploaded-content
ST_ALLOWED_UPLOAD_IMAGE_FORMAT = ('jpeg', 'gif')
# Only media types are allowed:
# https://www.iana.org/assignments/media-types/media-types.xhtml
ST_ALLOWED_UPLOAD_FILE_MEDIA_TYPE = OrderedDict([
('doc', 'application/msword'), # .doc
('docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'), # .docx
('pdf', 'application/pdf'),
])
ST_ALLOWED_URL_PROTOCOLS = {
'http', 'https', 'mailto', 'ftp', 'ftps',
'git', 'svn', 'magnet', 'irc', 'ircs'}

View File

@ -5,6 +5,7 @@ from __future__ import unicode_literals
from django.contrib.auth.decorators import login_required
from django.shortcuts import render, redirect, get_object_or_404
from django.http import HttpResponsePermanentRedirect
from django.conf import settings
from djconfig import config
@ -53,7 +54,8 @@ def publish(request, category_id=None):
context = {
'form': form,
'cform': cform}
'cform': cform,
}
return render(request, 'spirit/topic/publish.html', context)
@ -76,7 +78,9 @@ def update(request, pk):
else:
form = TopicForm(user=request.user, instance=topic)
context = {'form': form, }
context = {
'form': form,
}
return render(request, 'spirit/topic/update.html', context)