From 1d6045ec0d6066f2361f6cf36d66ae7afaafea07 Mon Sep 17 00:00:00 2001 From: Brandon Date: Wed, 12 Aug 2026 18:58:27 -0600 Subject: [PATCH 1/2] Add --rewrite-host option to resign_media_urls Media URLs are stored with an absolute scheme and host. When a self-hosted server's domain changes, previously stored timelapse, prediction, poster and event snapshot URLs keep pointing at the old host, and resign_media_urls could not repair them because new_signed_url() only replaces the query string. With --rewrite-host, absolute URLs are rewritten to the given host while being re-signed, with the scheme taken from SITE_USES_HTTPS (matching how the server builds all other full URLs). Relative URLs are left untouched. The digest is calculated from the path only, so rewriting the host never invalidates the signature. Without the flag, behavior is unchanged. --- .../management/commands/resign_media_urls.py | 29 +++++++--- backend/app/tests.py | 53 ++++++++++++++++++- backend/lib/tests.py | 40 +++++++++++++- backend/lib/url_signing.py | 10 +++- 4 files changed, 120 insertions(+), 12 deletions(-) diff --git a/backend/app/management/commands/resign_media_urls.py b/backend/app/management/commands/resign_media_urls.py index 0a6984d0d..def035a8a 100644 --- a/backend/app/management/commands/resign_media_urls.py +++ b/backend/app/management/commands/resign_media_urls.py @@ -31,8 +31,17 @@ def print_progress_bar(iteration, total, prefix='Progress:', suffix='Complete', class Command(BaseCommand): help = '(re-)signs all media URLs. Must be run once after updating, and any time the Django SECRET_KEY is rotated' + def add_arguments(self, parser): + parser.add_argument( + '--rewrite-host', + default=None, + help='Also rewrite absolute media URLs to point at this host, e.g. "obico.example.com". ' + 'Use after the server domain has changed. The scheme is taken from the SITE_USES_HTTPS setting. ' + 'Relative URLs are left untouched.' + ) + @staticmethod - def _resign_urls_on_model(obj: models.Model, url_fields: List[str]): + def _resign_urls_on_model(obj: models.Model, url_fields: List[str], rewrite_host=None): changed = False total_rows = len(obj.objects.all()) print(f"Resigning {obj.__name__} URLs ({total_rows} rows)...") @@ -40,7 +49,7 @@ def _resign_urls_on_model(obj: models.Model, url_fields: List[str]): for url_field in url_fields: url = getattr(row, url_field) if url: - setattr(row, url_field, new_signed_url(url)) + setattr(row, url_field, new_signed_url(url, rewrite_host=rewrite_host)) changed = True if changed: row.save() @@ -48,23 +57,27 @@ def _resign_urls_on_model(obj: models.Model, url_fields: List[str]): print_progress_bar(idx + 1, total_rows) print_progress_bar(1, 1) - def resign_urls(self): + def resign_urls(self, rewrite_host=None): self._resign_urls_on_model( obj=GCodeFile, # type: ignore - url_fields=['url', 'thumbnail1_url', 'thumbnail2_url', 'thumbnail3_url'] + url_fields=['url', 'thumbnail1_url', 'thumbnail2_url', 'thumbnail3_url'], + rewrite_host=rewrite_host ) self._resign_urls_on_model( obj=Print, # type: ignore - url_fields=['video_url', 'tagged_video_url', 'poster_url', 'prediction_json_url'] + url_fields=['video_url', 'tagged_video_url', 'poster_url', 'prediction_json_url'], + rewrite_host=rewrite_host ) self._resign_urls_on_model( obj=PrinterEvent, # type: ignore - url_fields=['image_url'] + url_fields=['image_url'], + rewrite_host=rewrite_host ) self._resign_urls_on_model( obj=PrintShotFeedback, # type: ignore - url_fields=['image_url'] + url_fields=['image_url'], + rewrite_host=rewrite_host ) def handle(self, *args, **options): - self.resign_urls() + self.resign_urls(rewrite_host=options['rewrite_host']) diff --git a/backend/app/tests.py b/backend/app/tests.py index 6fa356e7d..b63fd50f1 100644 --- a/backend/app/tests.py +++ b/backend/app/tests.py @@ -1,13 +1,15 @@ from django.contrib.sites.models import Site -from django.test import TestCase +from django.core.management import call_command +from django.test import TestCase, override_settings from django.utils import timezone from unittest.mock import ANY, patch, PropertyMock from requests import Response from requests.exceptions import HTTPError -from app.models import MobileDevice, Print, Printer, PrinterEvent, User +from app.models import GCodeFile, MobileDevice, Print, Printer, PrinterEvent, User from app.models.syndicate_models import Syndicate from lib import mobile_notifications +from lib.url_signing import HmacSignedUrl from lib.utils import get_rotated_pic_url @@ -156,3 +158,50 @@ def test_get_rotated_pic_url_reads_cached_pic_once(self): } self.assertEqual(get_rotated_pic_url(self.printer), img_url) + + +class ResignMediaUrlsCommandTestCase(TestCase): + + def setUp(self): + syndicate, _ = Syndicate.objects.get_or_create(id=1, defaults={'name': 'test'}) + self.user = User.objects.create(email='resign@test.com', syndicate=syndicate) + self.printer = Printer.objects.create(user=self.user) + self.print = Print.objects.create( + user=self.user, + printer=self.printer, + filename='test.gcode', + ext_id=1, + started_at=timezone.now(), + finished_at=timezone.now(), + video_url='http://old-host:3334/media/tsd-timelapses/private/1.mp4?digest=stale', + ) + self.gcode_file = GCodeFile.objects.create( + user=self.user, + filename='test.gcode', + safe_filename='test.gcode', + url='/media/g_code_files/1.gcode?digest=stale', + ) + + def test_without_rewrite_host_urls_keep_their_host(self): + call_command('resign_media_urls') + + self.print.refresh_from_db() + self.assertTrue(self.print.video_url.startswith( + 'http://old-host:3334/media/tsd-timelapses/private/1.mp4?digest=')) + self.assertTrue(HmacSignedUrl(self.print.video_url).is_authorized()) + + @override_settings(SITE_USES_HTTPS=False) + def test_rewrite_host_rewrites_absolute_urls_with_valid_digest(self): + call_command('resign_media_urls', rewrite_host='new-host') + + self.print.refresh_from_db() + self.assertTrue(self.print.video_url.startswith( + 'http://new-host/media/tsd-timelapses/private/1.mp4?digest=')) + self.assertTrue(HmacSignedUrl(self.print.video_url).is_authorized()) + + def test_rewrite_host_leaves_relative_urls_untouched(self): + call_command('resign_media_urls', rewrite_host='new-host') + + self.gcode_file.refresh_from_db() + self.assertTrue(self.gcode_file.url.startswith('/media/g_code_files/1.gcode?digest=')) + self.assertTrue(HmacSignedUrl(self.gcode_file.url).is_authorized()) diff --git a/backend/lib/tests.py b/backend/lib/tests.py index 6c60c727b..9c925414c 100644 --- a/backend/lib/tests.py +++ b/backend/lib/tests.py @@ -1,9 +1,10 @@ -from django.test import TransactionTestCase +from django.test import TestCase, TransactionTestCase, override_settings from unittest.mock import patch from app.models import User, HeaterTracker, Printer, Print from .heater_trackers import process_heater_temps +from .url_signing import HmacSignedUrl, new_signed_url class HeaterTrackerTestCase(TransactionTestCase): @@ -225,3 +226,40 @@ def test_heater_target_is_saved_when_first_reached(self): self.assertEqual(print.PrintHeaterTarget_set.first().name, 'h0') self.assertEqual(print.PrintHeaterTarget_set.first().target, 60.0) + + +class NewSignedUrlTestCase(TestCase): + + def test_host_is_preserved_without_rewrite_host(self): + signed = new_signed_url('http://old-host:3334/media/tsd-timelapses/private/1.mp4?digest=stale') + + self.assertTrue(signed.startswith('http://old-host:3334/media/tsd-timelapses/private/1.mp4?digest=')) + self.assertTrue(HmacSignedUrl(signed).is_authorized()) + + @override_settings(SITE_USES_HTTPS=False) + def test_rewrite_host_replaces_scheme_and_host(self): + signed = new_signed_url( + 'https://old-host:3334/media/tsd-timelapses/private/1.mp4?digest=stale', + rewrite_host='new-host') + + self.assertTrue(signed.startswith('http://new-host/media/tsd-timelapses/private/1.mp4?digest=')) + + @override_settings(SITE_USES_HTTPS=True) + def test_rewrite_host_uses_https_when_site_uses_https(self): + signed = new_signed_url( + 'http://old-host:3334/media/tsd-timelapses/private/1.mp4', + rewrite_host='new-host') + + self.assertTrue(signed.startswith('https://new-host/media/tsd-timelapses/private/1.mp4?digest=')) + + def test_rewrite_host_leaves_relative_url_untouched(self): + signed = new_signed_url('/media/g_code_files/1.gcode?digest=stale', rewrite_host='new-host') + + self.assertTrue(signed.startswith('/media/g_code_files/1.gcode?digest=')) + + def test_digest_is_valid_after_rewrite(self): + signed = new_signed_url( + 'http://old-host:3334/media/tsd-timelapses/private/1.mp4', + rewrite_host='new-host') + + self.assertTrue(HmacSignedUrl(signed).is_authorized()) diff --git a/backend/lib/url_signing.py b/backend/lib/url_signing.py index abbc0ddb4..bbcf39803 100644 --- a/backend/lib/url_signing.py +++ b/backend/lib/url_signing.py @@ -23,16 +23,24 @@ def calculate_hmac_digest(path: str): return base64.urlsafe_b64encode(str(digest).encode()).decode() -def new_signed_url(url_str: str) -> str: +def new_signed_url(url_str: str, rewrite_host: Optional[str] = None) -> str: """ Signs a URL based on a given Signature is appended in the form of URL parameters in the query string. Note that the entire query string will be replaced (everything after '?' in the URL). + + If rewrite_host is given, absolute URLs are rewritten to point at that host, with + the scheme taken from the SITE_USES_HTTPS setting. Relative URLs are left as-is. + The digest is calculated from the path only, so rewriting the host does not + invalidate the signature. """ parsed_url = urlparse(url_str) digest = calculate_hmac_digest(parsed_url.path) signed_url = parsed_url._replace(query=f"digest={digest}") + if rewrite_host and parsed_url.netloc: + scheme = 'https' if settings.SITE_USES_HTTPS else 'http' + signed_url = signed_url._replace(scheme=scheme, netloc=rewrite_host) return urlunparse(signed_url) From 696e98474230914fe7577b319d5fe64a933694ea Mon Sep 17 00:00:00 2001 From: Brandon Date: Sun, 16 Aug 2026 16:46:11 -0600 Subject: [PATCH 2/2] Rework as standalone rewrite_media_url_host command --- .../management/commands/resign_media_urls.py | 79 ++----------------- .../commands/rewrite_media_url_host.py | 32 ++++++++ backend/app/tests.py | 69 +++++++++++++--- backend/lib/media_urls.py | 61 ++++++++++++++ backend/lib/tests.py | 51 ++++++------ backend/lib/url_signing.py | 38 ++++++--- website/docs/server-guides/configure.md | 12 +++ 7 files changed, 226 insertions(+), 116 deletions(-) create mode 100644 backend/app/management/commands/rewrite_media_url_host.py create mode 100644 backend/lib/media_urls.py diff --git a/backend/app/management/commands/resign_media_urls.py b/backend/app/management/commands/resign_media_urls.py index def035a8a..646cbb790 100644 --- a/backend/app/management/commands/resign_media_urls.py +++ b/backend/app/management/commands/resign_media_urls.py @@ -1,83 +1,16 @@ -from typing import List - from django.core.management.base import BaseCommand -from app.models import Print, PrinterEvent, GCodeFile, models, PrintShotFeedback +from lib.media_urls import MEDIA_URL_FIELDS, transform_urls_on_model from lib.url_signing import new_signed_url -# https://stackoverflow.com/questions/3173320/text-progress-bar-in-terminal-with-block-characters -def print_progress_bar(iteration, total, prefix='Progress:', suffix='Complete', decimals=1, length=50, fill='X', printEnd=""): - """ - Call in a loop to create terminal progress bar - @params: - iteration - Required : current iteration (Int) - total - Required : total iterations (Int) - prefix - Optional : prefix string (Str) - suffix - Optional : suffix string (Str) - decimals - Optional : positive number of decimals in percent complete (Int) - length - Optional : character length of bar (Int) - fill - Optional : bar fill character (Str) - printEnd - Optional : end character (e.g. "\r", "\r\n") (Str) - """ - percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total))) - filledLength = int(length * iteration // total) - bar = fill * filledLength + '-' * (length - filledLength) - print(f'\r{prefix} |{bar}| {percent}% {suffix}', end=printEnd, flush=True) - # Print New Line on Complete - if iteration == total: - print() - class Command(BaseCommand): help = '(re-)signs all media URLs. Must be run once after updating, and any time the Django SECRET_KEY is rotated' - def add_arguments(self, parser): - parser.add_argument( - '--rewrite-host', - default=None, - help='Also rewrite absolute media URLs to point at this host, e.g. "obico.example.com". ' - 'Use after the server domain has changed. The scheme is taken from the SITE_USES_HTTPS setting. ' - 'Relative URLs are left untouched.' - ) - - @staticmethod - def _resign_urls_on_model(obj: models.Model, url_fields: List[str], rewrite_host=None): - changed = False - total_rows = len(obj.objects.all()) - print(f"Resigning {obj.__name__} URLs ({total_rows} rows)...") - for idx, row in enumerate(obj.objects.all()): - for url_field in url_fields: - url = getattr(row, url_field) - if url: - setattr(row, url_field, new_signed_url(url, rewrite_host=rewrite_host)) - changed = True - if changed: - row.save() - if idx % 20 == 0: - print_progress_bar(idx + 1, total_rows) - print_progress_bar(1, 1) - - def resign_urls(self, rewrite_host=None): - self._resign_urls_on_model( - obj=GCodeFile, # type: ignore - url_fields=['url', 'thumbnail1_url', 'thumbnail2_url', 'thumbnail3_url'], - rewrite_host=rewrite_host - ) - self._resign_urls_on_model( - obj=Print, # type: ignore - url_fields=['video_url', 'tagged_video_url', 'poster_url', 'prediction_json_url'], - rewrite_host=rewrite_host - ) - self._resign_urls_on_model( - obj=PrinterEvent, # type: ignore - url_fields=['image_url'], - rewrite_host=rewrite_host - ) - self._resign_urls_on_model( - obj=PrintShotFeedback, # type: ignore - url_fields=['image_url'], - rewrite_host=rewrite_host - ) + def resign_urls(self): + for obj, url_fields in MEDIA_URL_FIELDS: + print(f"Resigning {obj.__name__} URLs ({len(obj.objects.all())} rows)...") + transform_urls_on_model(obj, url_fields, new_signed_url) def handle(self, *args, **options): - self.resign_urls(rewrite_host=options['rewrite_host']) + self.resign_urls() diff --git a/backend/app/management/commands/rewrite_media_url_host.py b/backend/app/management/commands/rewrite_media_url_host.py new file mode 100644 index 000000000..4f02b0817 --- /dev/null +++ b/backend/app/management/commands/rewrite_media_url_host.py @@ -0,0 +1,32 @@ +from django.core.management.base import BaseCommand, CommandError + +from lib.media_urls import MEDIA_URL_FIELDS, transform_urls_on_model +from lib.url_signing import normalize_origin, rewrite_url_origin + + +class Command(BaseCommand): + help = ('Rewrites the scheme and host of all media URLs that point at --old-origin so they point at --new-origin instead. ' + 'Run this after the server address has changed, so that previously stored timelapses, snapshots, etc. keep working. ' + 'URLs on any other origin, and relative URLs, are left untouched. Query strings (including digests) are preserved.') + + def add_arguments(self, parser): + parser.add_argument('--old-origin', required=True, help='Origin currently stored in the media URLs, e.g. "http://192.168.1.50:3334"') + parser.add_argument('--new-origin', required=True, help='Origin to rewrite them to, e.g. "https://obico.example.com"') + parser.add_argument('--dry-run', action='store_true', help='Only report how many URLs would be rewritten. Nothing is saved.') + + def rewrite_urls(self, old_origin, new_origin, dry_run=False): + for obj, url_fields in MEDIA_URL_FIELDS: + print(f"Rewriting {obj.__name__} URLs ({len(obj.objects.all())} rows)...") + changed = transform_urls_on_model( + obj, url_fields, lambda url: rewrite_url_origin(url, old_origin, new_origin), save=not dry_run) + print(f"{obj.__name__}: {changed} URLs {'would be rewritten (dry run)' if dry_run else 'rewritten'}") + + def handle(self, *args, **options): + try: + old_origin = normalize_origin(options['old_origin']) + new_origin = normalize_origin(options['new_origin']) + except ValueError as e: + raise CommandError(str(e)) + if options['dry_run']: + print("Dry run: no changes will be saved.") + self.rewrite_urls(old_origin, new_origin, dry_run=options['dry_run']) diff --git a/backend/app/tests.py b/backend/app/tests.py index b63fd50f1..717819591 100644 --- a/backend/app/tests.py +++ b/backend/app/tests.py @@ -1,6 +1,7 @@ from django.contrib.sites.models import Site from django.core.management import call_command -from django.test import TestCase, override_settings +from django.core.management.base import CommandError +from django.test import TestCase from django.utils import timezone from unittest.mock import ANY, patch, PropertyMock from requests import Response @@ -9,7 +10,7 @@ from app.models import GCodeFile, MobileDevice, Print, Printer, PrinterEvent, User from app.models.syndicate_models import Syndicate from lib import mobile_notifications -from lib.url_signing import HmacSignedUrl +from lib.url_signing import HmacSignedUrl, new_signed_url from lib.utils import get_rotated_pic_url @@ -182,7 +183,7 @@ def setUp(self): url='/media/g_code_files/1.gcode?digest=stale', ) - def test_without_rewrite_host_urls_keep_their_host(self): + def test_urls_keep_their_host_and_get_valid_digest(self): call_command('resign_media_urls') self.print.refresh_from_db() @@ -190,18 +191,64 @@ def test_without_rewrite_host_urls_keep_their_host(self): 'http://old-host:3334/media/tsd-timelapses/private/1.mp4?digest=')) self.assertTrue(HmacSignedUrl(self.print.video_url).is_authorized()) - @override_settings(SITE_USES_HTTPS=False) - def test_rewrite_host_rewrites_absolute_urls_with_valid_digest(self): - call_command('resign_media_urls', rewrite_host='new-host') + self.gcode_file.refresh_from_db() + self.assertTrue(self.gcode_file.url.startswith('/media/g_code_files/1.gcode?digest=')) + self.assertTrue(HmacSignedUrl(self.gcode_file.url).is_authorized()) + + +class RewriteMediaUrlHostCommandTestCase(TestCase): + + def setUp(self): + syndicate, _ = Syndicate.objects.get_or_create(id=1, defaults={'name': 'test'}) + self.user = User.objects.create(email='rewrite@test.com', syndicate=syndicate) + self.printer = Printer.objects.create(user=self.user) + self.print = Print.objects.create( + user=self.user, + printer=self.printer, + filename='test.gcode', + ext_id=1, + started_at=timezone.now(), + finished_at=timezone.now(), + video_url=new_signed_url('http://old-host:3334/media/tsd-timelapses/private/1.mp4'), + poster_url='https://storage.example.com/bucket/1.jpg?sig=provider', + ) + self.gcode_file = GCodeFile.objects.create( + user=self.user, + filename='test.gcode', + safe_filename='test.gcode', + url=new_signed_url('/media/g_code_files/1.gcode'), + ) + + def test_matching_urls_are_rewritten_and_stay_valid(self): + old_video_url = self.print.video_url + + call_command('rewrite_media_url_host', old_origin='http://old-host:3334', new_origin='https://new-host') self.print.refresh_from_db() - self.assertTrue(self.print.video_url.startswith( - 'http://new-host/media/tsd-timelapses/private/1.mp4?digest=')) + self.assertEqual(self.print.video_url, old_video_url.replace('http://old-host:3334', 'https://new-host')) self.assertTrue(HmacSignedUrl(self.print.video_url).is_authorized()) - def test_rewrite_host_leaves_relative_urls_untouched(self): - call_command('resign_media_urls', rewrite_host='new-host') + def test_other_origin_and_relative_urls_are_untouched(self): + call_command('rewrite_media_url_host', old_origin='http://old-host:3334', new_origin='https://new-host') + self.print.refresh_from_db() self.gcode_file.refresh_from_db() + self.assertEqual(self.print.poster_url, 'https://storage.example.com/bucket/1.jpg?sig=provider') self.assertTrue(self.gcode_file.url.startswith('/media/g_code_files/1.gcode?digest=')) - self.assertTrue(HmacSignedUrl(self.gcode_file.url).is_authorized()) + + def test_invalid_origin_is_rejected_before_touching_anything(self): + old_video_url = self.print.video_url + + with self.assertRaises(CommandError): + call_command('rewrite_media_url_host', old_origin='old-host:3334', new_origin='https://new-host') + + self.print.refresh_from_db() + self.assertEqual(self.print.video_url, old_video_url) + + def test_dry_run_changes_nothing(self): + old_video_url = self.print.video_url + + call_command('rewrite_media_url_host', old_origin='http://old-host:3334', new_origin='https://new-host', dry_run=True) + + self.print.refresh_from_db() + self.assertEqual(self.print.video_url, old_video_url) diff --git a/backend/lib/media_urls.py b/backend/lib/media_urls.py new file mode 100644 index 000000000..da7adb16c --- /dev/null +++ b/backend/lib/media_urls.py @@ -0,0 +1,61 @@ +from typing import Callable, List + +from app.models import Print, PrinterEvent, GCodeFile, models, PrintShotFeedback + + +# All model fields that hold media URLs (timelapses, snapshots, thumbnails, etc.) +MEDIA_URL_FIELDS = [ + (GCodeFile, ['url', 'thumbnail1_url', 'thumbnail2_url', 'thumbnail3_url']), + (Print, ['video_url', 'tagged_video_url', 'poster_url', 'prediction_json_url']), + (PrinterEvent, ['image_url']), + (PrintShotFeedback, ['image_url']), +] + + +# https://stackoverflow.com/questions/3173320/text-progress-bar-in-terminal-with-block-characters +def print_progress_bar(iteration, total, prefix='Progress:', suffix='Complete', decimals=1, length=50, fill='X', printEnd=""): + """ + Call in a loop to create terminal progress bar + @params: + iteration - Required : current iteration (Int) + total - Required : total iterations (Int) + prefix - Optional : prefix string (Str) + suffix - Optional : suffix string (Str) + decimals - Optional : positive number of decimals in percent complete (Int) + length - Optional : character length of bar (Int) + fill - Optional : bar fill character (Str) + printEnd - Optional : end character (e.g. "\r", "\r\n") (Str) + """ + percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total))) + filledLength = int(length * iteration // total) + bar = fill * filledLength + '-' * (length - filledLength) + print(f'\r{prefix} |{bar}| {percent}% {suffix}', end=printEnd, flush=True) + # Print New Line on Complete + if iteration == total: + print() + + +def transform_urls_on_model(obj: models.Model, url_fields: List[str], transform: Callable[[str], str], save: bool = True) -> int: + """ + Applies `transform` to every non-empty URL field on every row of `obj`, saving each + row that changed. Returns the number of URLs that were changed. With save=False, + nothing is written but the count is still returned (for dry runs). + """ + changed_urls = 0 + total_rows = len(obj.objects.all()) + for idx, row in enumerate(obj.objects.all()): + changed = False + for url_field in url_fields: + url = getattr(row, url_field) + if url: + new_url = transform(url) + if new_url != url: + setattr(row, url_field, new_url) + changed = True + changed_urls += 1 + if changed and save: + row.save() + if idx % 20 == 0: + print_progress_bar(idx + 1, total_rows) + print_progress_bar(1, 1) + return changed_urls diff --git a/backend/lib/tests.py b/backend/lib/tests.py index 9c925414c..807e6dc26 100644 --- a/backend/lib/tests.py +++ b/backend/lib/tests.py @@ -1,10 +1,10 @@ -from django.test import TestCase, TransactionTestCase, override_settings +from django.test import TestCase, TransactionTestCase from unittest.mock import patch from app.models import User, HeaterTracker, Printer, Print from .heater_trackers import process_heater_temps -from .url_signing import HmacSignedUrl, new_signed_url +from .url_signing import HmacSignedUrl, new_signed_url, rewrite_url_origin class HeaterTrackerTestCase(TransactionTestCase): @@ -230,36 +230,41 @@ def test_heater_target_is_saved_when_first_reached(self): class NewSignedUrlTestCase(TestCase): - def test_host_is_preserved_without_rewrite_host(self): + def test_signed_url_keeps_scheme_host_and_path(self): signed = new_signed_url('http://old-host:3334/media/tsd-timelapses/private/1.mp4?digest=stale') self.assertTrue(signed.startswith('http://old-host:3334/media/tsd-timelapses/private/1.mp4?digest=')) self.assertTrue(HmacSignedUrl(signed).is_authorized()) - @override_settings(SITE_USES_HTTPS=False) - def test_rewrite_host_replaces_scheme_and_host(self): - signed = new_signed_url( - 'https://old-host:3334/media/tsd-timelapses/private/1.mp4?digest=stale', - rewrite_host='new-host') - self.assertTrue(signed.startswith('http://new-host/media/tsd-timelapses/private/1.mp4?digest=')) +class RewriteUrlOriginTestCase(TestCase): - @override_settings(SITE_USES_HTTPS=True) - def test_rewrite_host_uses_https_when_site_uses_https(self): - signed = new_signed_url( - 'http://old-host:3334/media/tsd-timelapses/private/1.mp4', - rewrite_host='new-host') + def test_matching_origin_is_rewritten_and_rest_of_url_is_preserved(self): + signed = new_signed_url('http://old-host:3334/media/tsd-timelapses/private/1.mp4') - self.assertTrue(signed.startswith('https://new-host/media/tsd-timelapses/private/1.mp4?digest=')) + rewritten = rewrite_url_origin(signed, 'http://old-host:3334', 'https://new-host') - def test_rewrite_host_leaves_relative_url_untouched(self): - signed = new_signed_url('/media/g_code_files/1.gcode?digest=stale', rewrite_host='new-host') + self.assertEqual(rewritten, signed.replace('http://old-host:3334', 'https://new-host')) + self.assertTrue(HmacSignedUrl(rewritten).is_authorized()) - self.assertTrue(signed.startswith('/media/g_code_files/1.gcode?digest=')) + def test_other_origin_is_untouched(self): + url = 'https://storage.example.com/bucket/1.mp4?sig=provider' - def test_digest_is_valid_after_rewrite(self): - signed = new_signed_url( - 'http://old-host:3334/media/tsd-timelapses/private/1.mp4', - rewrite_host='new-host') + self.assertEqual(rewrite_url_origin(url, 'http://old-host:3334', 'https://new-host'), url) - self.assertTrue(HmacSignedUrl(signed).is_authorized()) + def test_relative_url_is_untouched(self): + url = '/media/g_code_files/1.gcode?digest=abc' + + self.assertEqual(rewrite_url_origin(url, 'http://old-host:3334', 'https://new-host'), url) + + def test_origin_matching_ignores_case_and_trailing_slash(self): + rewritten = rewrite_url_origin( + 'http://OLD-Host:3334/media/1.mp4?digest=abc', 'http://old-host:3334/', 'https://New-Host/') + + self.assertEqual(rewritten, 'https://new-host/media/1.mp4?digest=abc') + + def test_invalid_origin_is_rejected(self): + with self.assertRaises(ValueError): + rewrite_url_origin('http://old-host:3334/media/1.mp4', 'old-host:3334', 'https://new-host') + with self.assertRaises(ValueError): + rewrite_url_origin('http://old-host:3334/media/1.mp4', 'http://old-host:3334', 'https://new-host/media') diff --git a/backend/lib/url_signing.py b/backend/lib/url_signing.py index bbcf39803..53a856382 100644 --- a/backend/lib/url_signing.py +++ b/backend/lib/url_signing.py @@ -23,27 +23,47 @@ def calculate_hmac_digest(path: str): return base64.urlsafe_b64encode(str(digest).encode()).decode() -def new_signed_url(url_str: str, rewrite_host: Optional[str] = None) -> str: +def new_signed_url(url_str: str) -> str: """ Signs a URL based on a given Signature is appended in the form of URL parameters in the query string. Note that the entire query string will be replaced (everything after '?' in the URL). - - If rewrite_host is given, absolute URLs are rewritten to point at that host, with - the scheme taken from the SITE_USES_HTTPS setting. Relative URLs are left as-is. - The digest is calculated from the path only, so rewriting the host does not - invalidate the signature. """ parsed_url = urlparse(url_str) digest = calculate_hmac_digest(parsed_url.path) signed_url = parsed_url._replace(query=f"digest={digest}") - if rewrite_host and parsed_url.netloc: - scheme = 'https' if settings.SITE_USES_HTTPS else 'http' - signed_url = signed_url._replace(scheme=scheme, netloc=rewrite_host) return urlunparse(signed_url) +def normalize_origin(origin_str: str) -> str: + """ + Normalizes an origin (scheme://host[:port]) for comparison: lower-cased, no trailing slash. + Raises ValueError if the string is not a valid origin. + """ + parsed = urlparse(origin_str.strip().rstrip('/')) + if not parsed.scheme or not parsed.netloc or parsed.path or parsed.query or parsed.fragment: + raise ValueError(f'"{origin_str}" is not a valid origin. Expected the form scheme://host[:port], e.g. https://obico.example.com') + return f'{parsed.scheme}://{parsed.netloc}'.lower() + + +def rewrite_url_origin(url_str: str, old_origin: str, new_origin: str) -> str: + """ + Rewrites the scheme and host of a URL if, and only if, they match old_origin. + + Everything else (path, query string, fragment) is preserved as-is, so a URL signed with + new_signed_url() stays valid because the digest is calculated from the path only. + Relative URLs and URLs on any other origin are returned unchanged. + """ + parsed_url = urlparse(url_str) + if not parsed_url.netloc: + return url_str + if f'{parsed_url.scheme}://{parsed_url.netloc}'.lower() != normalize_origin(old_origin): + return url_str + new_parsed = urlparse(normalize_origin(new_origin)) + return urlunparse(parsed_url._replace(scheme=new_parsed.scheme, netloc=new_parsed.netloc)) + + @dataclass class HmacSignedUrl: """This dataclass provides functions to check the validity of an HMAC signed url""" diff --git a/website/docs/server-guides/configure.md b/website/docs/server-guides/configure.md index 15d74cc38..ed225d74e 100644 --- a/website/docs/server-guides/configure.md +++ b/website/docs/server-guides/configure.md @@ -123,6 +123,18 @@ cd obico-server && docker compose run web ./manage.py site --fix ``` ::: +### 5. If you change the Django site later {#5-if-you-change-the-django-site-later} + +Links to timelapses, snapshots and other media are stored with the server address that was in use when they were created. If you later change the address (for example, moving from a LAN IP address to a domain name), those existing links will keep pointing at the old address and stop working. New prints are not affected. + +To fix the existing links, run the following command, replacing the two addresses with the old and the new address of your server, including `http://` or `https://` and the port if there is one: + +``` +cd obico-server && docker compose run web ./manage.py rewrite_media_url_host --old-origin http://old_server_ip_address:server_port --new-origin https://server_domain_name +``` + +Add `--dry-run` to see how many links would be changed without changing anything. + ## (Re-)generate `DJANGO_SECRET_KEY` {#re-generate-django_secret_key} This step is optional. But you are highly recommended to generate `DJANGO_SECRET_KEY` and rotate (re-generate) it periodically, especially if you have your Obico Server exposed to the Internet via reverse proxy.