Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 4 additions & 58 deletions backend/app/management/commands/resign_media_urls.py
Original file line number Diff line number Diff line change
@@ -1,70 +1,16 @@
from typing import List

from django.core.management.base import BaseCommand

from app.models import Print, PrinterEvent, GCodeFile, models, PrintShotFeedback
from lib.media_urls import MEDIA_URL_FIELDS, transform_urls_on_model
from lib.url_signing import new_signed_url


# https://stackoverflow.com/questions/3173320/text-progress-bar-in-terminal-with-block-characters
def print_progress_bar(iteration, total, prefix='Progress:', suffix='Complete', decimals=1, length=50, fill='X', printEnd=""):
"""
Call in a loop to create terminal progress bar
@params:
iteration - Required : current iteration (Int)
total - Required : total iterations (Int)
prefix - Optional : prefix string (Str)
suffix - Optional : suffix string (Str)
decimals - Optional : positive number of decimals in percent complete (Int)
length - Optional : character length of bar (Int)
fill - Optional : bar fill character (Str)
printEnd - Optional : end character (e.g. "\r", "\r\n") (Str)
"""
percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
filledLength = int(length * iteration // total)
bar = fill * filledLength + '-' * (length - filledLength)
print(f'\r{prefix} |{bar}| {percent}% {suffix}', end=printEnd, flush=True)
# Print New Line on Complete
if iteration == total:
print()

class Command(BaseCommand):
help = '(re-)signs all media URLs. Must be run once after updating, and any time the Django SECRET_KEY is rotated'

@staticmethod
def _resign_urls_on_model(obj: models.Model, url_fields: List[str]):
changed = False
total_rows = len(obj.objects.all())
print(f"Resigning {obj.__name__} URLs ({total_rows} rows)...")
for idx, row in enumerate(obj.objects.all()):
for url_field in url_fields:
url = getattr(row, url_field)
if url:
setattr(row, url_field, new_signed_url(url))
changed = True
if changed:
row.save()
if idx % 20 == 0:
print_progress_bar(idx + 1, total_rows)
print_progress_bar(1, 1)

def resign_urls(self):
self._resign_urls_on_model(
obj=GCodeFile, # type: ignore
url_fields=['url', 'thumbnail1_url', 'thumbnail2_url', 'thumbnail3_url']
)
self._resign_urls_on_model(
obj=Print, # type: ignore
url_fields=['video_url', 'tagged_video_url', 'poster_url', 'prediction_json_url']
)
self._resign_urls_on_model(
obj=PrinterEvent, # type: ignore
url_fields=['image_url']
)
self._resign_urls_on_model(
obj=PrintShotFeedback, # type: ignore
url_fields=['image_url']
)
for obj, url_fields in MEDIA_URL_FIELDS:
print(f"Resigning {obj.__name__} URLs ({len(obj.objects.all())} rows)...")
transform_urls_on_model(obj, url_fields, new_signed_url)

def handle(self, *args, **options):
self.resign_urls()
32 changes: 32 additions & 0 deletions backend/app/management/commands/rewrite_media_url_host.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
from django.core.management.base import BaseCommand, CommandError

from lib.media_urls import MEDIA_URL_FIELDS, transform_urls_on_model
from lib.url_signing import normalize_origin, rewrite_url_origin


class Command(BaseCommand):
help = ('Rewrites the scheme and host of all media URLs that point at --old-origin so they point at --new-origin instead. '
'Run this after the server address has changed, so that previously stored timelapses, snapshots, etc. keep working. '
'URLs on any other origin, and relative URLs, are left untouched. Query strings (including digests) are preserved.')

def add_arguments(self, parser):
parser.add_argument('--old-origin', required=True, help='Origin currently stored in the media URLs, e.g. "http://192.168.1.50:3334"')
parser.add_argument('--new-origin', required=True, help='Origin to rewrite them to, e.g. "https://obico.example.com"')
parser.add_argument('--dry-run', action='store_true', help='Only report how many URLs would be rewritten. Nothing is saved.')

def rewrite_urls(self, old_origin, new_origin, dry_run=False):
for obj, url_fields in MEDIA_URL_FIELDS:
print(f"Rewriting {obj.__name__} URLs ({len(obj.objects.all())} rows)...")
changed = transform_urls_on_model(
obj, url_fields, lambda url: rewrite_url_origin(url, old_origin, new_origin), save=not dry_run)
print(f"{obj.__name__}: {changed} URLs {'would be rewritten (dry run)' if dry_run else 'rewritten'}")

def handle(self, *args, **options):
try:
old_origin = normalize_origin(options['old_origin'])
new_origin = normalize_origin(options['new_origin'])
except ValueError as e:
raise CommandError(str(e))
if options['dry_run']:
print("Dry run: no changes will be saved.")
self.rewrite_urls(old_origin, new_origin, dry_run=options['dry_run'])
98 changes: 97 additions & 1 deletion backend/app/tests.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
from django.contrib.sites.models import Site
from django.core.management import call_command
from django.core.management.base import CommandError
from django.test import TestCase
from django.utils import timezone
from unittest.mock import ANY, patch, PropertyMock
from requests import Response
from requests.exceptions import HTTPError

from app.models import MobileDevice, Print, Printer, PrinterEvent, User
from app.models import GCodeFile, MobileDevice, Print, Printer, PrinterEvent, User
from app.models.syndicate_models import Syndicate
from lib import mobile_notifications
from lib.url_signing import HmacSignedUrl, new_signed_url
from lib.utils import get_rotated_pic_url


Expand Down Expand Up @@ -156,3 +159,96 @@ def test_get_rotated_pic_url_reads_cached_pic_once(self):
}

self.assertEqual(get_rotated_pic_url(self.printer), img_url)


class ResignMediaUrlsCommandTestCase(TestCase):

def setUp(self):
syndicate, _ = Syndicate.objects.get_or_create(id=1, defaults={'name': 'test'})
self.user = User.objects.create(email='resign@test.com', syndicate=syndicate)
self.printer = Printer.objects.create(user=self.user)
self.print = Print.objects.create(
user=self.user,
printer=self.printer,
filename='test.gcode',
ext_id=1,
started_at=timezone.now(),
finished_at=timezone.now(),
video_url='http://old-host:3334/media/tsd-timelapses/private/1.mp4?digest=stale',
)
self.gcode_file = GCodeFile.objects.create(
user=self.user,
filename='test.gcode',
safe_filename='test.gcode',
url='/media/g_code_files/1.gcode?digest=stale',
)

def test_urls_keep_their_host_and_get_valid_digest(self):
call_command('resign_media_urls')

self.print.refresh_from_db()
self.assertTrue(self.print.video_url.startswith(
'http://old-host:3334/media/tsd-timelapses/private/1.mp4?digest='))
self.assertTrue(HmacSignedUrl(self.print.video_url).is_authorized())

self.gcode_file.refresh_from_db()
self.assertTrue(self.gcode_file.url.startswith('/media/g_code_files/1.gcode?digest='))
self.assertTrue(HmacSignedUrl(self.gcode_file.url).is_authorized())


class RewriteMediaUrlHostCommandTestCase(TestCase):

def setUp(self):
syndicate, _ = Syndicate.objects.get_or_create(id=1, defaults={'name': 'test'})
self.user = User.objects.create(email='rewrite@test.com', syndicate=syndicate)
self.printer = Printer.objects.create(user=self.user)
self.print = Print.objects.create(
user=self.user,
printer=self.printer,
filename='test.gcode',
ext_id=1,
started_at=timezone.now(),
finished_at=timezone.now(),
video_url=new_signed_url('http://old-host:3334/media/tsd-timelapses/private/1.mp4'),
poster_url='https://storage.example.com/bucket/1.jpg?sig=provider',
)
self.gcode_file = GCodeFile.objects.create(
user=self.user,
filename='test.gcode',
safe_filename='test.gcode',
url=new_signed_url('/media/g_code_files/1.gcode'),
)

def test_matching_urls_are_rewritten_and_stay_valid(self):
old_video_url = self.print.video_url

call_command('rewrite_media_url_host', old_origin='http://old-host:3334', new_origin='https://new-host')

self.print.refresh_from_db()
self.assertEqual(self.print.video_url, old_video_url.replace('http://old-host:3334', 'https://new-host'))
self.assertTrue(HmacSignedUrl(self.print.video_url).is_authorized())

def test_other_origin_and_relative_urls_are_untouched(self):
call_command('rewrite_media_url_host', old_origin='http://old-host:3334', new_origin='https://new-host')

self.print.refresh_from_db()
self.gcode_file.refresh_from_db()
self.assertEqual(self.print.poster_url, 'https://storage.example.com/bucket/1.jpg?sig=provider')
self.assertTrue(self.gcode_file.url.startswith('/media/g_code_files/1.gcode?digest='))

def test_invalid_origin_is_rejected_before_touching_anything(self):
old_video_url = self.print.video_url

with self.assertRaises(CommandError):
call_command('rewrite_media_url_host', old_origin='old-host:3334', new_origin='https://new-host')

self.print.refresh_from_db()
self.assertEqual(self.print.video_url, old_video_url)

def test_dry_run_changes_nothing(self):
old_video_url = self.print.video_url

call_command('rewrite_media_url_host', old_origin='http://old-host:3334', new_origin='https://new-host', dry_run=True)

self.print.refresh_from_db()
self.assertEqual(self.print.video_url, old_video_url)
61 changes: 61 additions & 0 deletions backend/lib/media_urls.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
from typing import Callable, List

from app.models import Print, PrinterEvent, GCodeFile, models, PrintShotFeedback


# All model fields that hold media URLs (timelapses, snapshots, thumbnails, etc.)
MEDIA_URL_FIELDS = [
(GCodeFile, ['url', 'thumbnail1_url', 'thumbnail2_url', 'thumbnail3_url']),
(Print, ['video_url', 'tagged_video_url', 'poster_url', 'prediction_json_url']),
(PrinterEvent, ['image_url']),
(PrintShotFeedback, ['image_url']),
]


# https://stackoverflow.com/questions/3173320/text-progress-bar-in-terminal-with-block-characters
def print_progress_bar(iteration, total, prefix='Progress:', suffix='Complete', decimals=1, length=50, fill='X', printEnd=""):
"""
Call in a loop to create terminal progress bar
@params:
iteration - Required : current iteration (Int)
total - Required : total iterations (Int)
prefix - Optional : prefix string (Str)
suffix - Optional : suffix string (Str)
decimals - Optional : positive number of decimals in percent complete (Int)
length - Optional : character length of bar (Int)
fill - Optional : bar fill character (Str)
printEnd - Optional : end character (e.g. "\r", "\r\n") (Str)
"""
percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
filledLength = int(length * iteration // total)
bar = fill * filledLength + '-' * (length - filledLength)
print(f'\r{prefix} |{bar}| {percent}% {suffix}', end=printEnd, flush=True)
# Print New Line on Complete
if iteration == total:
print()


def transform_urls_on_model(obj: models.Model, url_fields: List[str], transform: Callable[[str], str], save: bool = True) -> int:
"""
Applies `transform` to every non-empty URL field on every row of `obj`, saving each
row that changed. Returns the number of URLs that were changed. With save=False,
nothing is written but the count is still returned (for dry runs).
"""
changed_urls = 0
total_rows = len(obj.objects.all())
for idx, row in enumerate(obj.objects.all()):
changed = False
for url_field in url_fields:
url = getattr(row, url_field)
if url:
new_url = transform(url)
if new_url != url:
setattr(row, url_field, new_url)
changed = True
changed_urls += 1
if changed and save:
row.save()
if idx % 20 == 0:
print_progress_bar(idx + 1, total_rows)
print_progress_bar(1, 1)
return changed_urls
45 changes: 44 additions & 1 deletion backend/lib/tests.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
from django.test import TransactionTestCase
from django.test import TestCase, TransactionTestCase
from unittest.mock import patch


from app.models import User, HeaterTracker, Printer, Print
from .heater_trackers import process_heater_temps
from .url_signing import HmacSignedUrl, new_signed_url, rewrite_url_origin


class HeaterTrackerTestCase(TransactionTestCase):
Expand Down Expand Up @@ -225,3 +226,45 @@ def test_heater_target_is_saved_when_first_reached(self):

self.assertEqual(print.PrintHeaterTarget_set.first().name, 'h0')
self.assertEqual(print.PrintHeaterTarget_set.first().target, 60.0)


class NewSignedUrlTestCase(TestCase):

def test_signed_url_keeps_scheme_host_and_path(self):
signed = new_signed_url('http://old-host:3334/media/tsd-timelapses/private/1.mp4?digest=stale')

self.assertTrue(signed.startswith('http://old-host:3334/media/tsd-timelapses/private/1.mp4?digest='))
self.assertTrue(HmacSignedUrl(signed).is_authorized())


class RewriteUrlOriginTestCase(TestCase):

def test_matching_origin_is_rewritten_and_rest_of_url_is_preserved(self):
signed = new_signed_url('http://old-host:3334/media/tsd-timelapses/private/1.mp4')

rewritten = rewrite_url_origin(signed, 'http://old-host:3334', 'https://new-host')

self.assertEqual(rewritten, signed.replace('http://old-host:3334', 'https://new-host'))
self.assertTrue(HmacSignedUrl(rewritten).is_authorized())

def test_other_origin_is_untouched(self):
url = 'https://storage.example.com/bucket/1.mp4?sig=provider'

self.assertEqual(rewrite_url_origin(url, 'http://old-host:3334', 'https://new-host'), url)

def test_relative_url_is_untouched(self):
url = '/media/g_code_files/1.gcode?digest=abc'

self.assertEqual(rewrite_url_origin(url, 'http://old-host:3334', 'https://new-host'), url)

def test_origin_matching_ignores_case_and_trailing_slash(self):
rewritten = rewrite_url_origin(
'http://OLD-Host:3334/media/1.mp4?digest=abc', 'http://old-host:3334/', 'https://New-Host/')

self.assertEqual(rewritten, 'https://new-host/media/1.mp4?digest=abc')

def test_invalid_origin_is_rejected(self):
with self.assertRaises(ValueError):
rewrite_url_origin('http://old-host:3334/media/1.mp4', 'old-host:3334', 'https://new-host')
with self.assertRaises(ValueError):
rewrite_url_origin('http://old-host:3334/media/1.mp4', 'http://old-host:3334', 'https://new-host/media')
28 changes: 28 additions & 0 deletions backend/lib/url_signing.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,34 @@ def new_signed_url(url_str: str) -> str:
return urlunparse(signed_url)


def normalize_origin(origin_str: str) -> str:
"""
Normalizes an origin (scheme://host[:port]) for comparison: lower-cased, no trailing slash.
Raises ValueError if the string is not a valid origin.
"""
parsed = urlparse(origin_str.strip().rstrip('/'))
if not parsed.scheme or not parsed.netloc or parsed.path or parsed.query or parsed.fragment:
raise ValueError(f'"{origin_str}" is not a valid origin. Expected the form scheme://host[:port], e.g. https://obico.example.com')
return f'{parsed.scheme}://{parsed.netloc}'.lower()


def rewrite_url_origin(url_str: str, old_origin: str, new_origin: str) -> str:
"""
Rewrites the scheme and host of a URL if, and only if, they match old_origin.

Everything else (path, query string, fragment) is preserved as-is, so a URL signed with
new_signed_url() stays valid because the digest is calculated from the path only.
Relative URLs and URLs on any other origin are returned unchanged.
"""
parsed_url = urlparse(url_str)
if not parsed_url.netloc:
return url_str
if f'{parsed_url.scheme}://{parsed_url.netloc}'.lower() != normalize_origin(old_origin):
return url_str
new_parsed = urlparse(normalize_origin(new_origin))
return urlunparse(parsed_url._replace(scheme=new_parsed.scheme, netloc=new_parsed.netloc))


@dataclass
class HmacSignedUrl:
"""This dataclass provides functions to check the validity of an HMAC signed url"""
Expand Down
12 changes: 12 additions & 0 deletions website/docs/server-guides/configure.md
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,18 @@ cd obico-server && docker compose run web ./manage.py site --fix
```
:::

### 5. If you change the Django site later {#5-if-you-change-the-django-site-later}

Links to timelapses, snapshots and other media are stored with the server address that was in use when they were created. If you later change the address (for example, moving from a LAN IP address to a domain name), those existing links will keep pointing at the old address and stop working. New prints are not affected.

To fix the existing links, run the following command, replacing the two addresses with the old and the new address of your server, including `http://` or `https://` and the port if there is one:

```
cd obico-server && docker compose run web ./manage.py rewrite_media_url_host --old-origin http://old_server_ip_address:server_port --new-origin https://server_domain_name
```

Add `--dry-run` to see how many links would be changed without changing anything.

## (Re-)generate `DJANGO_SECRET_KEY` {#re-generate-django_secret_key}

This step is optional. But you are highly recommended to generate `DJANGO_SECRET_KEY` and rotate (re-generate) it periodically, especially if you have your Obico Server exposed to the Internet via reverse proxy.
Expand Down