feat: Add automatic charset=UTF-8 to text content types

sanic/mixins/static.py

@@ -1,7 +1,6 @@
 from collections.abc import Sequence
 from email.utils import formatdate
 from functools import partial, wraps
-from mimetypes import guess_type
 from os import PathLike, path
 from pathlib import Path, PurePath
 from typing import Optional, Union
@@ -11,7 +10,6 @@
 
 from sanic.base.meta import SanicMeta
 from sanic.compat import stat_async
-from sanic.constants import DEFAULT_HTTP_CONTENT_TYPE
 from sanic.exceptions import FileNotFound, HeaderNotFound, RangeNotSatisfiable
 from sanic.handlers import ContentRangeHandler
 from sanic.handlers.directory import DirectoryHandler
@@ -20,6 +18,7 @@
 from sanic.models.futures import FutureStatic
 from sanic.request import Request
 from sanic.response import HTTPResponse, file, file_stream, validate_file
+from sanic.response.convenience import guess_content_type
 
 
 class StaticMixin(BaseMixin, metaclass=SanicMeta):
@@ -300,11 +299,7 @@ async def _static_request_handler(
                         headers.update(_range.headers)
 
             if "content-type" not in headers:
-                content_type = (
-                    content_type
-                    or guess_type(file_path)[0]
-                    or DEFAULT_HTTP_CONTENT_TYPE
-                )
+                content_type = content_type or guess_content_type(file_path)
 
                 if "charset=" not in content_type and (
                     content_type.startswith("text/")

sanic/response/convenience.py

@@ -290,12 +290,14 @@ async def file(
         else:
             out_stream = await f.read()
 
-    mime_type = mime_type or guess_type(filename)[0] or "text/plain"
+    content_type = mime_type or guess_content_type(
+        filename, fallback="text/plain; charset=utf-8"
+    )
     return HTTPResponse(
         body=out_stream,
         status=status,
         headers=headers,
-        content_type=mime_type,
+        content_type=content_type,
     )
 
 
@@ -395,3 +397,16 @@ async def _streaming_fn(response):
         headers=headers,
         content_type=mime_type,
     )
+
+
+def guess_content_type(
+    file_path: Union[str, PurePath],
+    fallback: str = DEFAULT_HTTP_CONTENT_TYPE,
+) -> str:
+    """Guess the content type (rather than MIME only) by the file extension."""
+    mediatype = guess_type(file_path)[0]
+    if mediatype is None:
+        return fallback
+    if mediatype.startswith("text/"):
+        return f"{mediatype}; charset=utf-8"
+    return mediatype

tests/test_response_file.py

@@ -4,7 +4,7 @@
 import pytest
 
 from sanic.compat import Header
-from sanic.response.convenience import validate_file
+from sanic.response.convenience import guess_content_type, validate_file
 
 
 @pytest.mark.parametrize(
@@ -53,3 +53,36 @@ async def test_file_timestamp_validation(
     else:
         record = records[0]
         assert expected in record.message
+
+
+@pytest.mark.parametrize(
+    "file_path,expected",
+    (
+        ("test.html", "text/html; charset=utf-8"),
+        ("test.txt", "text/plain; charset=utf-8"),
+        ("test.css", "text/css; charset=utf-8"),
+        ("test.js", "text/javascript; charset=utf-8"),
+        ("test.csv", "text/csv; charset=utf-8"),
+        ("test.xml", "application/xml"),
+        # Fallback for unknown types
+        ("test.file", "application/octet-stream"),
+    ),
+)
+def test_guess_content_type(file_path, expected):
+    """Test that guess_content_type correctly adds charset for text types."""
+    result = guess_content_type(file_path)
+    assert result == expected
+
+
+def test_guess_content_type_with_custom_fallback():
+    """Test that guess_content_type uses custom fallback for unknown types."""
+    result = guess_content_type("no_extension", fallback="custom/type")
+    assert result == "custom/type"
+
+
+def test_guess_content_type_with_pathlib():
+    """Test that guess_content_type works with pathlib Path objects."""
+    from pathlib import Path
+
+    result = guess_content_type(Path("test.html"))
+    assert result == "text/html; charset=utf-8"

tests/test_static.py

@@ -5,11 +5,13 @@
 from collections import Counter
 from pathlib import Path
 from time import gmtime, strftime
+from urllib.parse import unquote
 
 import pytest
 
 from sanic import Sanic, text
 from sanic.exceptions import FileNotFound, ServerError
+from sanic.response import file
 
 
 @pytest.fixture(scope="module")
@@ -150,43 +152,53 @@ def test_static_file_invalid_path(app, static_file_directory, file_name):
     assert response.status == 404
 
 
-@pytest.mark.parametrize("file_name", ["test.html"])
-def test_static_file_content_type(app, static_file_directory, file_name):
-    app.static(
-        "/testing.file",
-        get_file_path(static_file_directory, file_name),
-        content_type="text/html; charset=utf-8",
-    )
+@pytest.mark.parametrize(
+    "file_name,expected_content_type",
+    [
+        ("decode me.txt", "text/plain; charset=utf-8"),
+        ("test.html", "text/html; charset=utf-8"),
+        ("python.png", "image/png"),
+        # Note: file() default for unknown types differs from app.static
+        ("test.file", "text/plain; charset=utf-8"),
+    ],
+)
+def test_file_response_content_type(
+    app: Sanic, file_name, expected_content_type, static_file_directory
+):
+    """Responses by file() rather than app.static."""
 
-    request, response = app.test_client.get("/testing.file")
+    @app.get("/files/<filename>")
+    def file_route(request, filename):
+        file_path = os.path.join(static_file_directory, filename)
+        file_path = os.path.abspath(unquote(file_path))
+        return file(file_path)
+
+    request, response = app.test_client.get(f"/files/{file_name}")
     assert response.status == 200
-    assert response.body == get_file_content(static_file_directory, file_name)
-    assert response.headers["Content-Type"] == "text/html; charset=utf-8"
+    assert response.headers["Content-Type"] == expected_content_type
 
 
 @pytest.mark.parametrize(
-    "file_name,expected",
+    "file_name,expected_content_type",
     [
-        ("test.html", "text/html; charset=utf-8"),
         ("decode me.txt", "text/plain; charset=utf-8"),
+        ("test.html", "text/html; charset=utf-8"),
+        ("python.png", "image/png"),
         ("test.file", "application/octet-stream"),
     ],
 )
-def test_static_file_content_type_guessed(
-    app, static_file_directory, file_name, expected
+def test_static_file_content_type(
+    app: Sanic, file_name, expected_content_type, static_file_directory
 ):
-    app.static(
-        "/testing.file",
-        get_file_path(static_file_directory, file_name),
-    )
+    """Test that file responses automatically include charset for text."""
 
-    request, response = app.test_client.get("/testing.file")
+    app.static("/", static_file_directory)
+    request, response = app.test_client.get(file_name)
     assert response.status == 200
-    assert response.body == get_file_content(static_file_directory, file_name)
-    assert response.headers["Content-Type"] == expected
+    assert response.headers["Content-Type"] == expected_content_type
 
 
-def test_static_file_content_type_with_charset(app, static_file_directory):
+def test_static_file_content_type_forced(app, static_file_directory):
     app.static(
         "/testing.file",
         get_file_path(static_file_directory, "decode me.txt"),

tests/test_static_directory.py

@@ -35,7 +35,7 @@ def test_static_index_single(app: Sanic, static_file_directory: str):
     assert response.body == get_file_content(
         static_file_directory, "test.html"
     )
-    assert response.headers["Content-Type"] == "text/html"
+    assert response.headers["Content-Type"] == "text/html; charset=utf-8"
 
 
 def test_static_index_single_not_found(app: Sanic, static_file_directory: str):
@@ -57,7 +57,7 @@ def test_static_index_multiple(app: Sanic, static_file_directory: str):
     assert response.body == get_file_content(
         static_file_directory, "test.html"
     )
-    assert response.headers["Content-Type"] == "text/html"
+    assert response.headers["Content-Type"] == "text/html; charset=utf-8"
 
 
 def test_static_directory_view_and_index(
@@ -80,7 +80,7 @@ def test_static_directory_view_and_index(
     assert response.body == get_file_content(
         f"{static_file_directory}/nested/dir", "foo.txt"
     )
-    assert response.content_type == "text/plain"
+    assert response.content_type == "text/plain; charset=utf-8"
 
 
 def test_static_directory_handler(app: Sanic, static_file_directory: str):
@@ -102,7 +102,7 @@ def test_static_directory_handler(app: Sanic, static_file_directory: str):
     assert response.body == get_file_content(
         f"{static_file_directory}/nested/dir", "foo.txt"
     )
-    assert response.content_type == "text/plain"
+    assert response.content_type == "text/plain; charset=utf-8"
 
 
 def test_static_directory_handler_fails(app: Sanic):