forked from bexem/PlexCache
-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathfile_operations.py
More file actions
executable file
·440 lines (346 loc) · 19.4 KB
/
Copy pathfile_operations.py
File metadata and controls
executable file
·440 lines (346 loc) · 19.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
"""
File operations for PlexCache.
Handles file moving, filtering, subtitle operations, and path modifications.
"""
import os
import logging
from concurrent.futures import ThreadPoolExecutor
from typing import List, Set, Optional, Tuple
class FilePathModifier:
"""Handles file path modifications and conversions."""
def __init__(self, plex_source: str, real_source: str,
plex_library_folders: List[str], nas_library_folders: List[str]):
self.plex_source = plex_source
self.real_source = real_source
self.plex_library_folders = plex_library_folders
self.nas_library_folders = nas_library_folders
def modify_file_paths(self, files: List[str]) -> List[str]:
"""Modify file paths from Plex paths to real system paths."""
if files is None:
return []
logging.info("Editing file paths...")
# Filter the files based on those that start with the plex_source path
files = [file_path for file_path in files if file_path.startswith(self.plex_source)]
# Iterate over each file path and modify it accordingly
for i, file_path in enumerate(files):
logging.info(f"Original path: {file_path}")
# Replace the plex_source with the real_source in the file path
file_path = file_path.replace(self.plex_source, self.real_source, 1)
# Determine which library folder is in the file path
for j, folder in enumerate(self.plex_library_folders):
if folder in file_path:
# Replace the plex library folder with the corresponding NAS library folder
file_path = file_path.replace(folder, self.nas_library_folders[j])
break
# Update the modified file path in the files list
files[i] = file_path
logging.info(f"Edited path: {file_path}")
return files or []
class SubtitleFinder:
"""Handles subtitle file discovery and operations."""
def __init__(self, subtitle_extensions: Optional[List[str]] = None):
if subtitle_extensions is None:
subtitle_extensions = [".srt", ".vtt", ".sbv", ".sub", ".idx"]
self.subtitle_extensions = subtitle_extensions
def get_media_subtitles(self, media_files: List[str], files_to_skip: Optional[Set[str]] = None) -> List[str]:
"""Get subtitle files for media files."""
logging.info("Fetching subtitles...")
files_to_skip = set() if files_to_skip is None else set(files_to_skip)
processed_files = set()
all_media_files = media_files.copy()
for file in media_files:
if file in files_to_skip or file in processed_files:
continue
processed_files.add(file)
directory_path = os.path.dirname(file)
if os.path.exists(directory_path):
subtitle_files = self._find_subtitle_files(directory_path, file)
all_media_files.extend(subtitle_files)
for subtitle_file in subtitle_files:
logging.info(f"Subtitle found: {subtitle_file}")
return all_media_files or []
def _find_subtitle_files(self, directory_path: str, file: str) -> List[str]:
"""Find subtitle files in a directory for a given media file."""
file_name, _ = os.path.splitext(os.path.basename(file))
try:
subtitle_files = [
entry.path
for entry in os.scandir(directory_path)
if entry.is_file() and entry.name.startswith(file_name) and
entry.name != file and entry.name.endswith(tuple(self.subtitle_extensions))
]
except PermissionError as e:
logging.error(f"Cannot access directory {directory_path}. Permission denied. Error: {e}")
subtitle_files = []
except OSError as e:
logging.error(f"Cannot access directory {directory_path}. Error: {e}")
subtitle_files = []
return subtitle_files or []
class FileFilter:
"""Handles file filtering based on destination and conditions."""
def __init__(self, real_source: str, cache_dir: str, is_unraid: bool,
mover_cache_exclude_file: str):
self.real_source = real_source
self.cache_dir = cache_dir
self.is_unraid = is_unraid
self.mover_cache_exclude_file = mover_cache_exclude_file or ""
def filter_files(self, files: List[str], destination: str,
media_to_cache: Optional[List[str]] = None,
files_to_skip: Optional[Set[str]] = None) -> List[str]:
"""Filter files based on destination and conditions."""
if media_to_cache is None:
media_to_cache = []
processed_files = set()
media_to = []
cache_files_to_exclude = []
if not files:
return []
for file in files:
if file in processed_files or (files_to_skip and file in files_to_skip):
continue
processed_files.add(file)
cache_file_name = self._get_cache_paths(file)[1]
cache_files_to_exclude.append(cache_file_name)
if destination == 'array':
if self._should_add_to_array(file, cache_file_name, media_to_cache):
media_to.append(file)
logging.info(f"Adding file to array: {file}")
elif destination == 'cache':
if self._should_add_to_cache(file, cache_file_name):
media_to.append(file)
logging.info(f"Adding file to cache: {file}")
return media_to or []
def _should_add_to_array(self, file: str, cache_file_name: str, media_to_cache: List[str]) -> bool:
"""Determine if a file should be added to the array."""
if file in media_to_cache:
return False
array_file = file.replace("/mnt/user/", "/mnt/user0/", 1) if self.is_unraid else file
if os.path.isfile(array_file):
# File already exists in the array
if os.path.isfile(cache_file_name):
os.remove(cache_file_name)
logging.info(f"Removed cache version of file: {cache_file_name}")
return False # No need to add to array
return True # Otherwise, the file should be added to the array
def _should_add_to_cache(self, file: str, cache_file_name: str) -> bool:
"""Determine if a file should be added to the cache."""
array_file = file.replace("/mnt/user/", "/mnt/user0/", 1) if self.is_unraid else file
if os.path.isfile(cache_file_name) and os.path.isfile(array_file):
# Uncomment the following line if you want to remove the array version when the file exists in the cache
os.remove(array_file)
logging.info(f"Removed array version of file: {array_file}")
return False
return not os.path.isfile(cache_file_name)
def _get_cache_paths(self, file: str) -> Tuple[str, str]:
"""Get cache path and filename for a given file."""
# Get the cache path by replacing the real source directory with the cache directory
cache_path = os.path.dirname(file).replace(self.real_source, self.cache_dir, 1)
# Get the cache file name by joining the cache path with the base name of the file
cache_file_name = os.path.join(cache_path, os.path.basename(file))
return cache_path, cache_file_name
def get_files_to_move_back_to_array(self, current_ondeck_items: Set[str],
current_watchlist_items: Set[str]) -> Tuple[List[str], List[str]]:
"""Get files in cache that should be moved back to array because they're no longer needed."""
files_to_move_back = []
cache_paths_to_remove = []
try:
# Read the exclude file to get all files currently in cache
if not os.path.exists(self.mover_cache_exclude_file):
logging.info("No exclude file found, nothing to move back")
return files_to_move_back, cache_paths_to_remove
with open(self.mover_cache_exclude_file, 'r') as f:
cache_files = [line.strip() for line in f if line.strip()]
logging.info(f"Found {len(cache_files)} files in exclude list")
# Get shows that are still needed (in OnDeck or watchlist)
needed_shows = set()
for item in current_ondeck_items | current_watchlist_items:
# Extract show name from path (e.g., "House Hunters (1999)" from "/path/to/House Hunters (1999) {imdb-tt0369117}/Season 263/...")
show_name = self._extract_show_name(item)
if show_name:
needed_shows.add(show_name)
# Check each file in cache
for cache_file in cache_files:
if not os.path.exists(cache_file):
logging.debug(f"Cache file no longer exists: {cache_file}")
cache_paths_to_remove.append(cache_file)
continue
# Extract show name from cache file
show_name = self._extract_show_name(cache_file)
if not show_name:
continue
# If show is still needed, keep this file in cache
if show_name in needed_shows:
logging.debug(f"Show still needed, keeping in cache: {show_name}")
continue
# Show is no longer needed, move this file back to array
array_file = cache_file.replace(self.cache_dir, self.real_source, 1)
logging.info(f"Show no longer needed, will move back to array: {show_name} - {cache_file}")
files_to_move_back.append(array_file)
cache_paths_to_remove.append(cache_file)
logging.info(f"Found {len(files_to_move_back)} files to move back to array")
except Exception as e:
logging.error(f"Error getting files to move back to array: {str(e)}")
return files_to_move_back, cache_paths_to_remove
def _extract_show_name(self, file_path: str) -> str:
"""Extract show name from file path."""
try:
# Split path and find the show directory (usually the last directory before Season)
path_parts = file_path.split('/')
for i, part in enumerate(path_parts):
if part.startswith('Season') or part.isdigit():
if i > 0:
return path_parts[i-1]
break
return ""
except Exception:
return ""
def remove_files_from_exclude_list(self, cache_paths_to_remove: List[str]) -> None:
"""Remove specified files from the exclude list."""
try:
if not os.path.exists(self.mover_cache_exclude_file):
logging.warning("Exclude file does not exist, cannot remove files")
return
# Read current exclude list
with open(self.mover_cache_exclude_file, 'r') as f:
current_files = [line.strip() for line in f if line.strip()]
# Remove specified files
updated_files = [f for f in current_files if f not in cache_paths_to_remove]
# Write back updated list
with open(self.mover_cache_exclude_file, 'w') as f:
for file_path in updated_files:
f.write(f"{file_path}\n")
logging.info(f"Removed {len(cache_paths_to_remove)} files from exclude list")
except Exception as e:
logging.error(f"Error removing files from exclude list: {str(e)}")
class FileMover:
"""Handles file moving operations."""
def __init__(self, real_source: str, cache_dir: str, is_unraid: bool,
file_utils, debug: bool = False, mover_cache_exclude_file: Optional[str] = None):
self.real_source = real_source
self.cache_dir = cache_dir
self.is_unraid = is_unraid
self.file_utils = file_utils
self.debug = debug
self.mover_cache_exclude_file = mover_cache_exclude_file
def move_media_files(self, files: List[str], destination: str,
max_concurrent_moves_array: int, max_concurrent_moves_cache: int) -> None:
"""Move media files to the specified destination."""
logging.info(f"Moving media files to {destination}...")
logging.debug(f"Total files to process: {len(files)}")
processed_files = set()
move_commands = []
cache_file_names = []
# Iterate over each file to move
for file_to_move in files:
if file_to_move in processed_files:
continue
processed_files.add(file_to_move)
# Get the user path, cache path, cache file name, and user file name
user_path, cache_path, cache_file_name, user_file_name = self._get_paths(file_to_move)
# Get the move command for the current file
move = self._get_move_command(destination, cache_file_name, user_path, user_file_name, cache_path)
if move is not None:
move_commands.append((move, cache_file_name))
logging.debug(f"Added move command for: {file_to_move}")
else:
logging.debug(f"No move command generated for: {file_to_move}")
logging.info(f"Generated {len(move_commands)} move commands for {destination}")
# Execute the move commands
self._execute_move_commands(move_commands, max_concurrent_moves_array,
max_concurrent_moves_cache, destination)
def _get_paths(self, file_to_move: str) -> Tuple[str, str, str, str]:
"""Get all necessary paths for file moving."""
# Get the user path
user_path = os.path.dirname(file_to_move)
# Get the relative path from the real source directory
relative_path = os.path.relpath(user_path, self.real_source)
# Get the cache path by joining the cache directory with the relative path
cache_path = os.path.join(self.cache_dir, relative_path)
# Get the cache file name by joining the cache path with the base name of the file to move
cache_file_name = os.path.join(cache_path, os.path.basename(file_to_move))
# Modify the user path if unraid is True
if self.is_unraid:
user_path = user_path.replace("/mnt/user/", "/mnt/user0/", 1)
# Get the user file name by joining the user path with the base name of the file to move
user_file_name = os.path.join(user_path, os.path.basename(file_to_move))
return user_path, cache_path, cache_file_name, user_file_name
def _get_move_command(self, destination: str, cache_file_name: str,
user_path: str, user_file_name: str, cache_path: str) -> Optional[Tuple[str, str]]:
"""Get the move command for a file."""
move = None
if destination == 'array':
self.file_utils.create_directory_with_permissions(user_path, cache_file_name)
if os.path.isfile(cache_file_name):
move = (cache_file_name, user_path)
elif destination == 'cache':
self.file_utils.create_directory_with_permissions(cache_path, user_file_name)
if not os.path.isfile(cache_file_name):
move = (user_file_name, cache_path)
return move
def _execute_move_commands(self, move_commands: List[Tuple[Tuple[str, str], str]],
max_concurrent_moves_array: int, max_concurrent_moves_cache: int,
destination: str) -> None:
"""Execute the move commands."""
if self.debug:
for move_cmd, cache_file_name in move_commands:
logging.info(move_cmd)
else:
max_concurrent_moves = max_concurrent_moves_array if destination == 'array' else max_concurrent_moves_cache
from functools import partial
with ThreadPoolExecutor(max_workers=max_concurrent_moves) as executor:
results = list(executor.map(partial(self._move_file, destination=destination), move_commands))
errors = [result for result in results if result != 0]
logging.info(f"Finished moving files with {len(errors)} errors.")
def _move_file(self, move_cmd_with_cache: Tuple[Tuple[str, str], str], destination: str) -> int:
"""Move a single file and update exclude file if moving to cache."""
(src, dest), cache_file_name = move_cmd_with_cache
try:
self.file_utils.move_file(src, dest)
logging.info(f"Moved file from {src} to {dest} with original permissions and owner.")
# Only append to exclude file if moving to cache and move succeeded
if destination == 'cache' and self.mover_cache_exclude_file:
with open(self.mover_cache_exclude_file, "a") as f:
f.write(f"{cache_file_name}\n")
return 0
except Exception as e:
logging.error(f"Error moving file: {str(e)}")
return 1
class CacheCleanup:
"""Handles cleanup of empty folders in cache directories."""
def __init__(self, cache_dir: str):
self.cache_dir = cache_dir
def cleanup_empty_folders(self) -> None:
"""Remove empty folders from cache directories."""
logging.info("Starting cache cleanup process...")
cleaned_count = 0
# Check both tv and movies directories
for subdir in ['tv', 'movies']:
subdir_path = os.path.join(self.cache_dir, subdir)
if os.path.exists(subdir_path):
logging.debug(f"Cleaning up {subdir} directory: {subdir_path}")
cleaned_count += self._cleanup_directory(subdir_path)
else:
logging.debug(f"Directory does not exist, skipping: {subdir_path}")
if cleaned_count > 0:
logging.info(f"Cleaned up {cleaned_count} empty folders")
else:
logging.info("No empty folders found to clean up")
def _cleanup_directory(self, directory_path: str) -> int:
"""Recursively remove empty folders from a directory."""
cleaned_count = 0
try:
# Walk through the directory tree from bottom up
for root, dirs, files in os.walk(directory_path, topdown=False):
for dir_name in dirs:
dir_path = os.path.join(root, dir_name)
try:
# Check if directory is empty
if not os.listdir(dir_path):
os.rmdir(dir_path)
logging.debug(f"Removed empty folder: {dir_path}")
cleaned_count += 1
except OSError as e:
logging.debug(f"Could not remove directory {dir_path}: {e}")
except Exception as e:
logging.error(f"Error cleaning up directory {directory_path}: {e}")
return cleaned_count