feat: add optional staging directory for temp outputs · 5626192

feat: add optional staging directory for temp outputs
Browse files

bogdan committed a month ago

main

1 parent ccc23d8

commit 5626192

Showing 3 changed files with 64 additions and 5 deletions

+2 -0

CHANGELOG.md

@@ -12,6 +12,8 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 - `--continue-on-error` renamed to `--keep-going` (old name kept as alias)
 ### Added
 - `--unattended` preset (`--delete-source` + `--keep-going`) for long unattended runs
+- `--staging-dir DIR` to place intermediate transcoding temp files on a fast local path,
+	with automatic fallback to destination temp files when staging is unavailable
 
 - `cleanup_garmin_varia_media_folder.sh` helper for media-folder hygiene (import + transcoded output):
 	removes AppleDouble artifacts (`._*`), removes zero-size MP4 files,


+5 -0

README.md

View

@@ -28,6 +28,9 @@ manifest placeholder for a future FIT sync pipeline.
 # Long unattended run preset (delete-source + keep-going)
 ./garmin_varia_transcode.sh -s SampleFootage -d Output --unattended
 
+# NFS destination with local staging for temp outputs
+./garmin_varia_transcode.sh -s SampleFootage -d ~/Autofs/xdev/is-baobab/nvme0n1/@backup/Garmin --staging-dir /tmp/varia_staging
+
 # Media cleanup helper (Apple artifacts, zero-byte MP4, duplicate suffix normalization)
 ./cleanup_garmin_varia_media_folder.sh --dry-run ~/Autofs/xdev/autonas/ext01/@Camera/import
 ```
@@ -49,6 +52,8 @@ manifest placeholder for a future FIT sync pipeline.
 Options:
   -s, --source, --input DIR       Source directory or single file (default: current directory)
   -d, --destination, --output DIR Destination directory (default: current directory)
+  --staging-dir DIR               Temporary staging directory for intermediate output files
+                                  Falls back to destination temp files if staging cannot be used
   --mode MODE                     hardware|auto|quality|compat (default: hardware)
   --crf N                         CRF for quality/compat modes (lower = better; default: 20/19)
   --no-overwrite                  Skip files that already exist at destination


+57 -5

garmin_varia_transcode.sh

View

@@ -28,6 +28,7 @@ GARMIN_INFERRED_FOCAL_35MM_MM="7"
 
 SOURCE_DIR="$DEFAULT_SOURCE"
 DEST_DIR="$DEFAULT_SOURCE"
+STAGING_DIR=""
 MODE="$DEFAULT_MODE"
 EXTENSIONS_CSV="$DEFAULT_EXTENSIONS"
 CRF_OVERRIDE=""
@@ -39,6 +40,7 @@ VERBOSE=false
 MOVE_SOURCE=false
 SOURCE_PROVIDED=false
 DEST_PROVIDED=false
+STAGING_PROVIDED=false
 FAIL_FAST=true
 SOURCE_READABLE_MODE="normal"
 APPLE_REPACK_FALLBACK=true
@@ -81,6 +83,8 @@ Usage:
 Options:
   -s, --source, --input DIR       Source directory or single file (default: current directory)
   -d, --destination, --output DIR Destination directory (default: current directory)
+  --staging-dir DIR               Temporary staging directory for intermediate output files
+                                  (falls back to destination if staging cannot be used)
   --mode MODE                     Encoding mode (default: hardware); see Encoding Modes below
   --crf N                         CRF value for quality/compat modes (lower = better; default: 20/19)
   --no-overwrite                  Skip files that already exist at destination (default: overwrite)
@@ -319,20 +323,39 @@ make_temp_log_file() {
 
 make_temp_output_file() {
   local target_file="$1"
-  local target_dir target_base target_noext temp_path seed_path
+  local preferred_dir="${2:-}"
+  local target_dir target_base target_noext temp_path seed_path work_dir
 
   target_dir="$(dirname "$target_file")"
   target_base="$(basename "$target_file")"
   target_noext="${target_base%.*}"
+  work_dir="$target_dir"
+
+  if [[ -n "$preferred_dir" ]]; then
+    work_dir="$preferred_dir"
+  fi
+
+  seed_path="$(mktemp "$work_dir/.varia_tmp.${target_noext}.XXXXXX" 2>/dev/null || true)"
+  if [[ -z "$seed_path" && "$work_dir" != "$target_dir" ]]; then
+    seed_path="$(mktemp "$target_dir/.varia_tmp.${target_noext}.XXXXXX" 2>/dev/null || true)"
+  fi
 
-  seed_path="$(mktemp "$target_dir/.varia_tmp.${target_noext}.XXXXXX" 2>/dev/null || true)"
   if [[ -n "$seed_path" ]]; then
     rm -f -- "$seed_path" 2>/dev/null || true
     temp_path="${seed_path}.mp4"
   else
-    temp_path="$target_dir/.varia_tmp.${target_noext}.$$.$RANDOM.mp4"
+    temp_path="$work_dir/.varia_tmp.${target_noext}.$$.$RANDOM.mp4"
     if ! touch "$temp_path" >/dev/null 2>&1; then
-      temp_path=""
+      if [[ "$work_dir" != "$target_dir" ]]; then
+        temp_path="$target_dir/.varia_tmp.${target_noext}.$$.$RANDOM.mp4"
+        if ! touch "$temp_path" >/dev/null 2>&1; then
+          temp_path=""
+        else
+          rm -f -- "$temp_path" 2>/dev/null || true
+        fi
+      else
+        temp_path=""
+      fi
     else
       rm -f -- "$temp_path" 2>/dev/null || true
     fi
@@ -344,8 +367,11 @@ make_temp_output_file() {
 cleanup_transcode_artifacts() {
   local temp_output="$1"
   local final_output="$2"
+  local exif_backup_default="${temp_output}_original"
+  local exif_backup_alt="${temp_output}.exif_original"
 
   rm -f -- "$temp_output" 2>/dev/null || true
+  rm -f -- "$exif_backup_default" "$exif_backup_alt" 2>/dev/null || true
 
   # Defensive cleanup for previously failed non-atomic runs.
   if [[ -f "$final_output" && ! -s "$final_output" ]]; then
@@ -414,6 +440,12 @@ parse_args() {
         DEST_PROVIDED=true
         shift 2
         ;;
+      --staging-dir)
+        require_value "$1" "${2:-}"
+        STAGING_DIR="$2"
+        STAGING_PROVIDED=true
+        shift 2
+        ;;
       --mode)
         require_value "$1" "${2:-}"
         MODE="$2"
@@ -925,6 +957,22 @@ normalize_dest_dir() {
   DEST_DIR="$(to_abs_path "$DEST_DIR")"
 }
 
+normalize_staging_dir() {
+  if [[ "$STAGING_PROVIDED" != true ]]; then
+    STAGING_DIR=""
+    return
+  fi
+
+  STAGING_DIR="$(to_abs_path "$STAGING_DIR")"
+  [[ -d "$STAGING_DIR" ]] || die "Staging directory not found: $STAGING_DIR"
+  STAGING_DIR="$(cd "$STAGING_DIR" && pwd)"
+  [[ -w "$STAGING_DIR" ]] || die "Staging directory not writable: $STAGING_DIR"
+
+  if path_is_within "$STAGING_DIR" "$SOURCE_DIR"; then
+    die "Staging directory must not be inside source: staging=$STAGING_DIR source=$SOURCE_DIR"
+  fi
+}
+
 collect_extensions() {
   local raw="$EXTENSIONS_CSV"
   local token
@@ -1053,13 +1101,16 @@ process_video_file() {
 
   build_video_args
 
-  temp_output_file="$(make_temp_output_file "$output_file")"
+  temp_output_file="$(make_temp_output_file "$output_file" "$STAGING_DIR")"
   if [[ -z "$temp_output_file" ]]; then
     ERRORS=$((ERRORS + 1))
     DESTINATION_FAILURES=$((DESTINATION_FAILURES + 1))
     log_msg "ERROR" "Could not create temporary output file: $output_file"
     return 3
   fi
+  if [[ "$STAGING_PROVIDED" == true ]] && ! path_is_within "$temp_output_file" "$STAGING_DIR"; then
+    log_msg "WARN" "Staging unavailable for temp output; using destination directory: $temp_output_file"
+  fi
 
   local cmd=(ffmpeg -hide_banner)
   if [[ "$SOURCE_READABLE_MODE" == "tolerant" ]]; then
@@ -1397,6 +1448,7 @@ main() {
 
   normalize_source_dir
   normalize_dest_dir
+  normalize_staging_dir
   collect_extensions
 
   if path_is_within "$DEST_DIR" "$SOURCE_DIR"; then


	@@ -28,6 +28,7 @@ GARMIN_INFERRED_FOCAL_35MM_MM="7"
28	28
29	29	SOURCE_DIR="$DEFAULT_SOURCE"
30	30	DEST_DIR="$DEFAULT_SOURCE"
	31	+STAGING_DIR=""
31	32	MODE="$DEFAULT_MODE"
32	33	EXTENSIONS_CSV="$DEFAULT_EXTENSIONS"
33	34	CRF_OVERRIDE=""
	@@ -39,6 +40,7 @@ VERBOSE=false
39	40	MOVE_SOURCE=false
40	41	SOURCE_PROVIDED=false
41	42	DEST_PROVIDED=false
	43	+STAGING_PROVIDED=false
42	44	FAIL_FAST=true
43	45	SOURCE_READABLE_MODE="normal"
44	46	APPLE_REPACK_FALLBACK=true
	@@ -81,6 +83,8 @@ Usage:
81	83	Options:
82	84	-s, --source, --input DIR Source directory or single file (default: current directory)
83	85	-d, --destination, --output DIR Destination directory (default: current directory)
	86	+ --staging-dir DIR Temporary staging directory for intermediate output files
	87	+ (falls back to destination if staging cannot be used)
84	88	--mode MODE Encoding mode (default: hardware); see Encoding Modes below
85	89	--crf N CRF value for quality/compat modes (lower = better; default: 20/19)
86	90	--no-overwrite Skip files that already exist at destination (default: overwrite)
	@@ -319,20 +323,39 @@ make_temp_log_file() {
319	323
320	324	make_temp_output_file() {
321	325	local target_file="$1"
322		- local target_dir target_base target_noext temp_path seed_path
	326	+ local preferred_dir="${2:-}"
	327	+ local target_dir target_base target_noext temp_path seed_path work_dir
323	328
324	329	target_dir="$(dirname "$target_file")"
325	330	target_base="$(basename "$target_file")"
326	331	target_noext="${target_base%.*}"
	332	+ work_dir="$target_dir"
	333	+
	334	+ if [[ -n "$preferred_dir" ]]; then
	335	+ work_dir="$preferred_dir"
	336	+ fi
	337	+
	338	+ seed_path="$(mktemp "$work_dir/.varia_tmp.${target_noext}.XXXXXX" 2>/dev/null \|\| true)"
	339	+ if [[ -z "$seed_path" && "$work_dir" != "$target_dir" ]]; then
	340	+ seed_path="$(mktemp "$target_dir/.varia_tmp.${target_noext}.XXXXXX" 2>/dev/null \|\| true)"
	341	+ fi
327	342
328		- seed_path="$(mktemp "$target_dir/.varia_tmp.${target_noext}.XXXXXX" 2>/dev/null \|\| true)"
329	343	if [[ -n "$seed_path" ]]; then
330	344	rm -f -- "$seed_path" 2>/dev/null \|\| true
331	345	temp_path="${seed_path}.mp4"
332	346	else
333		- temp_path="$target_dir/.varia_tmp.${target_noext}.$$.$RANDOM.mp4"
	347	+ temp_path="$work_dir/.varia_tmp.${target_noext}.$$.$RANDOM.mp4"
334	348	if ! touch "$temp_path" >/dev/null 2>&1; then
335		- temp_path=""
	349	+ if [[ "$work_dir" != "$target_dir" ]]; then
	350	+ temp_path="$target_dir/.varia_tmp.${target_noext}.$$.$RANDOM.mp4"
	351	+ if ! touch "$temp_path" >/dev/null 2>&1; then
	352	+ temp_path=""
	353	+ else
	354	+ rm -f -- "$temp_path" 2>/dev/null \|\| true
	355	+ fi
	356	+ else
	357	+ temp_path=""
	358	+ fi
336	359	else
337	360	rm -f -- "$temp_path" 2>/dev/null \|\| true
338	361	fi
	@@ -344,8 +367,11 @@ make_temp_output_file() {
344	367	cleanup_transcode_artifacts() {
345	368	local temp_output="$1"
346	369	local final_output="$2"
	370	+ local exif_backup_default="${temp_output}_original"
	371	+ local exif_backup_alt="${temp_output}.exif_original"
347	372
348	373	rm -f -- "$temp_output" 2>/dev/null \|\| true
	374	+ rm -f -- "$exif_backup_default" "$exif_backup_alt" 2>/dev/null \|\| true
349	375
350	376	# Defensive cleanup for previously failed non-atomic runs.
351	377	if [[ -f "$final_output" && ! -s "$final_output" ]]; then
	@@ -414,6 +440,12 @@ parse_args() {
414	440	DEST_PROVIDED=true
415	441	shift 2
416	442	;;
	443	+ --staging-dir)
	444	+ require_value "$1" "${2:-}"
	445	+ STAGING_DIR="$2"
	446	+ STAGING_PROVIDED=true
	447	+ shift 2
	448	+ ;;
417	449	--mode)
418	450	require_value "$1" "${2:-}"
419	451	MODE="$2"
	@@ -925,6 +957,22 @@ normalize_dest_dir() {
925	957	DEST_DIR="$(to_abs_path "$DEST_DIR")"
926	958	}
927	959
	960	+normalize_staging_dir() {
	961	+ if [[ "$STAGING_PROVIDED" != true ]]; then
	962	+ STAGING_DIR=""
	963	+ return
	964	+ fi
	965	+
	966	+ STAGING_DIR="$(to_abs_path "$STAGING_DIR")"
	967	+ [[ -d "$STAGING_DIR" ]] \|\| die "Staging directory not found: $STAGING_DIR"
	968	+ STAGING_DIR="$(cd "$STAGING_DIR" && pwd)"
	969	+ [[ -w "$STAGING_DIR" ]] \|\| die "Staging directory not writable: $STAGING_DIR"
	970	+
	971	+ if path_is_within "$STAGING_DIR" "$SOURCE_DIR"; then
	972	+ die "Staging directory must not be inside source: staging=$STAGING_DIR source=$SOURCE_DIR"
	973	+ fi
	974	+}
	975	+
928	976	collect_extensions() {
929	977	local raw="$EXTENSIONS_CSV"
930	978	local token
	@@ -1053,13 +1101,16 @@ process_video_file() {
1053	1101
1054	1102	build_video_args
1055	1103
1056		- temp_output_file="$(make_temp_output_file "$output_file")"
	1104	+ temp_output_file="$(make_temp_output_file "$output_file" "$STAGING_DIR")"
1057	1105	if [[ -z "$temp_output_file" ]]; then
1058	1106	ERRORS=$((ERRORS + 1))
1059	1107	DESTINATION_FAILURES=$((DESTINATION_FAILURES + 1))
1060	1108	log_msg "ERROR" "Could not create temporary output file: $output_file"
1061	1109	return 3
1062	1110	fi
	1111	+ if [[ "$STAGING_PROVIDED" == true ]] && ! path_is_within "$temp_output_file" "$STAGING_DIR"; then
	1112	+ log_msg "WARN" "Staging unavailable for temp output; using destination directory: $temp_output_file"
	1113	+ fi
1063	1114
1064	1115	local cmd=(ffmpeg -hide_banner)
1065	1116	if [[ "$SOURCE_READABLE_MODE" == "tolerant" ]]; then
	@@ -1397,6 +1448,7 @@ main() {
1397	1448
1398	1449	normalize_source_dir
1399	1450	normalize_dest_dir
	1451	+ normalize_staging_dir
1400	1452	collect_extensions
1401	1453
1402	1454	if path_is_within "$DEST_DIR" "$SOURCE_DIR"; then

	@@ -12,6 +12,8 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
12	12	- `--continue-on-error` renamed to `--keep-going` (old name kept as alias)
13	13	### Added
14	14	- `--unattended` preset (`--delete-source` + `--keep-going`) for long unattended runs
	15	+- `--staging-dir DIR` to place intermediate transcoding temp files on a fast local path,
	16	+ with automatic fallback to destination temp files when staging is unavailable
15	17
16	18	- `cleanup_garmin_varia_media_folder.sh` helper for media-folder hygiene (import + transcoded output):
17	19	removes AppleDouble artifacts (`._*`), removes zero-size MP4 files,

	@@ -28,6 +28,9 @@ manifest placeholder for a future FIT sync pipeline.
28	28	# Long unattended run preset (delete-source + keep-going)
29	29	./garmin_varia_transcode.sh -s SampleFootage -d Output --unattended
30	30
	31	+# NFS destination with local staging for temp outputs
	32	+./garmin_varia_transcode.sh -s SampleFootage -d ~/Autofs/xdev/is-baobab/nvme0n1/@backup/Garmin --staging-dir /tmp/varia_staging
	33	+
31	34	# Media cleanup helper (Apple artifacts, zero-byte MP4, duplicate suffix normalization)
32	35	./cleanup_garmin_varia_media_folder.sh --dry-run ~/Autofs/xdev/autonas/ext01/@Camera/import
33	36	```
	@@ -49,6 +52,8 @@ manifest placeholder for a future FIT sync pipeline.
49	52	Options:
50	53	-s, --source, --input DIR Source directory or single file (default: current directory)
51	54	-d, --destination, --output DIR Destination directory (default: current directory)
	55	+ --staging-dir DIR Temporary staging directory for intermediate output files
	56	+ Falls back to destination temp files if staging cannot be used
52	57	--mode MODE hardware\|auto\|quality\|compat (default: hardware)
53	58	--crf N CRF for quality/compat modes (lower = better; default: 20/19)
54	59	--no-overwrite Skip files that already exist at destination