Clean up import report metrics · 7059ffc

Clean up import report metrics
Browse files

bogdan committed 2 weeks ago

main

1 parent 8b9540d

commit 7059ffc

Showing 3 changed files with 119 additions and 30 deletions

+9 -0

DEVELOPMENT.md

@@ -276,6 +276,15 @@ The test runner includes automatic result persistence:
 
 ---
 
+### 2026-05-18 08:40 - Report Table Cleanup
+
+- Aligned final report labels with a shared formatter.
+- Removed file-per-time throughput and average-per-file metrics from the final report.
+- Kept elapsed time and data throughput in `MB/sec`, which better reflects transfer performance.
+- Made test report cleanup portable on macOS by removing GNU `find -printf` and negative `head` usage.
+
+---
+
 ### 2026-05-13 21:10 - Critical Data Loss Incident + Hardening
 
 - Observed incident: running with explicit source on a problematic/near-full card led to writes on source media and source content removal without strong post-write confirmation.


+91 -21

media-importer.sh

View

@@ -207,18 +207,72 @@ get_file_size() {
     fi
 }
 
-# (Removed checksum/prefix/conflict helper functions to revert to pre-conflict-resolution behavior)
-
 # Safe move/copy helpers: filter out benign "set flags (was: ...): Operation not supported"
 # which appears when moving files onto filesystems that don't support BSD file flags
 # (macOS mv may try to preserve flags and print this warning while still succeeding).
 safe_mv() {
     local src="$1" dst="$2"
+    if [[ -e "$dst" ]]; then
+        log_message "Refusing to overwrite existing destination: $dst" "ERROR"
+        return 1
+    fi
     # Redirect stderr through a filter that removes the known benign message
     mv "$src" "$dst" 2> >(grep -v "set flags (was:" >&2)
     return $?
 }
 
+safe_cp() {
+    local src="$1" dst="$2"
+    if [[ -e "$dst" ]]; then
+        log_message "Refusing to overwrite existing destination: $dst" "ERROR"
+        return 1
+    fi
+    cp -p "$src" "$dst" 2> >(grep -v "set flags (was:" >&2)
+    return $?
+}
+
+ensure_unique_destination_path() {
+    local desired_path="$1"
+
+    if [[ -z "$desired_path" ]]; then
+        return 1
+    fi
+
+    if ! destination_path_unavailable "$desired_path"; then
+        echo "$desired_path"
+        return 0
+    fi
+
+    local dir_path filename ext stem candidate
+    dir_path=$(dirname "$desired_path")
+    filename=$(basename "$desired_path")
+
+    if [[ "$filename" == *.* ]]; then
+        ext="${filename##*.}"
+        stem="${filename%.*}"
+    else
+        ext=""
+        stem="$filename"
+    fi
+
+    local i
+    i=1
+    while [[ $i -le 9999 ]]; do
+        if [[ -n "$ext" ]]; then
+            candidate="$dir_path/${stem}_${i}.${ext}"
+        else
+            candidate="$dir_path/${stem}_${i}"
+        fi
+        if ! destination_path_unavailable "$candidate"; then
+            echo "$candidate"
+            return 0
+        fi
+        i=$((i + 1))
+    done
+
+    return 1
+}
+
 destination_path_reserved() {
     local candidate="$1"
     local reserved_path
@@ -647,30 +701,28 @@ format_duration() {
     fi
 }
 
-format_processing_rate() {
-    local files_count="$1"
-    local bytes_count="$2"
-    local elapsed_seconds="$3"
+format_data_rate() {
+    local bytes_count="$1"
+    local elapsed_seconds="$2"
 
-    awk -v files="$files_count" -v bytes="$bytes_count" -v seconds="$elapsed_seconds" '
+    awk -v bytes="$bytes_count" -v seconds="$elapsed_seconds" '
         BEGIN {
-            if (seconds <= 0 || files <= 0) {
+            if (seconds <= 0 || bytes <= 0) {
                 exit
             }
 
-            files_per_second = files / seconds
-            files_per_minute = files * 60 / seconds
             mb_per_second = bytes / seconds / 1048576
-
-            if (files_per_second >= 1) {
-                printf "%.2f files/sec, %.2f MB/sec", files_per_second, mb_per_second
-            } else {
-                printf "%.2f files/min, %.2f MB/sec", files_per_minute, mb_per_second
-            }
+            printf "%.2f MB/sec", mb_per_second
         }
     '
 }
 
+report_line() {
+    local label="$1"
+    local value="$2"
+    printf "  %-24s %s\n" "$label" "$value"
+}
+
 # Function to extract date from file
 extract_file_date() {
     local file="$1"
@@ -1171,13 +1223,31 @@ show_report() {
     print_color "$GREEN" "           PROCESSING REPORT"
     print_color "$GREEN" "=========================================="
     echo ""
-    
+
+    echo "Files Summary:"
+    report_line "Total files found:" "$TOTAL_FILES"
+    report_line "Successfully processed:" "$PROCESSED_FILES"
+    report_line "Skipped:" "$SKIPPED_FILES"
+    report_line "Errors:" "$ERROR_FILES"
+    echo ""
+
     echo "Size Summary:"
-    printf "  %-22s %s\n" "Total size found:" "$(format_size $TOTAL_SIZE)"
-    printf "  %-22s %s\n" "Successfully processed:" "$(format_size $PROCESSED_SIZE)"
-    
+    report_line "Total size found:" "$(format_size $TOTAL_SIZE)"
+    report_line "Successfully processed:" "$(format_size $PROCESSED_SIZE)"
     echo ""
-    
+
+    echo "Time Summary:"
+    report_line "Time elapsed:" "$(printf "%02d:%02d:%02d" $hours $minutes $seconds)"
+    if [[ $elapsed_time -gt 0 && $PROCESSED_SIZE -gt 0 ]]; then
+        local data_rate
+        data_rate=$(format_data_rate "$PROCESSED_SIZE" "$elapsed_time")
+        if [[ -n "$data_rate" ]]; then
+            report_line "Data rate:" "$data_rate"
+        fi
+    fi
+
+    echo ""
+
     if [[ $DRY_RUN -eq 1 ]]; then
         print_color "$YELLOW" "DRY RUN MODE - No files were actually moved/copied"
     elif [[ $KEEP_ORIGINALS -eq 1 ]]; then


+19 -9

test_runner.sh

View

@@ -206,19 +206,29 @@ EOF
 cleanup_old_reports() {
     if [[ -d "$TEST_REPORTS_DIR" ]]; then
         # Count total reports
-        local total_reports=$(find "$TEST_REPORTS_DIR" -name "*.md" | wc -l)
+        local total_reports
+        total_reports=$(find "$TEST_REPORTS_DIR" -name "*.md" | wc -l | tr -d ' ')
 
         if [[ $total_reports -gt 10 ]]; then
             print_color "$BLUE" "Cleaning up old test reports (keeping last 10)..."
 
-            # Find and remove oldest reports, keeping the 10 most recent
-            find "$TEST_REPORTS_DIR" -name "*.md" -type f -printf '%T@ %p\n' | \
-                sort -n | \
-                head -n -$((10)) | \
-                cut -d' ' -f2- | \
-                xargs -r rm
-
-            local remaining=$(find "$TEST_REPORTS_DIR" -name "*.md" | wc -l)
+            local remove_count
+            remove_count=$((total_reports - 10))
+
+            find "$TEST_REPORTS_DIR" -name "*.md" -type f | while IFS= read -r report_file; do
+                local report_mtime
+                if report_mtime=$(stat -f %m "$report_file" 2>/dev/null); then
+                    :
+                else
+                    report_mtime=$(stat -c %Y "$report_file" 2>/dev/null || echo 0)
+                fi
+                printf '%s\t%s\n' "$report_mtime" "$report_file"
+            done | sort -n | head -n "$remove_count" | cut -f2- | while IFS= read -r old_report; do
+                rm -f "$old_report"
+            done
+
+            local remaining
+            remaining=$(find "$TEST_REPORTS_DIR" -name "*.md" | wc -l | tr -d ' ')
             print_color "$GREEN" "Kept $remaining most recent test reports"
         fi
     fi


	@@ -276,6 +276,15 @@ The test runner includes automatic result persistence:
276	276
277	277	---
278	278
	279	+### 2026-05-18 08:40 - Report Table Cleanup
	280	+
	281	+- Aligned final report labels with a shared formatter.
	282	+- Removed file-per-time throughput and average-per-file metrics from the final report.
	283	+- Kept elapsed time and data throughput in `MB/sec`, which better reflects transfer performance.
	284	+- Made test report cleanup portable on macOS by removing GNU `find -printf` and negative `head` usage.
	285	+
	286	+---
	287	+
279	288	### 2026-05-13 21:10 - Critical Data Loss Incident + Hardening
280	289
281	290	- Observed incident: running with explicit source on a problematic/near-full card led to writes on source media and source content removal without strong post-write confirmation.

	@@ -207,18 +207,72 @@ get_file_size() {
207	207	fi
208	208	}
209	209
210		-# (Removed checksum/prefix/conflict helper functions to revert to pre-conflict-resolution behavior)
211		-
212	210	# Safe move/copy helpers: filter out benign "set flags (was: ...): Operation not supported"
213	211	# which appears when moving files onto filesystems that don't support BSD file flags
214	212	# (macOS mv may try to preserve flags and print this warning while still succeeding).
215	213	safe_mv() {
216	214	local src="$1" dst="$2"
	215	+ if [[ -e "$dst" ]]; then
	216	+ log_message "Refusing to overwrite existing destination: $dst" "ERROR"
	217	+ return 1
	218	+ fi
217	219	# Redirect stderr through a filter that removes the known benign message
218	220	mv "$src" "$dst" 2> >(grep -v "set flags (was:" >&2)
219	221	return $?
220	222	}
221	223
	224	+safe_cp() {
	225	+ local src="$1" dst="$2"
	226	+ if [[ -e "$dst" ]]; then
	227	+ log_message "Refusing to overwrite existing destination: $dst" "ERROR"
	228	+ return 1
	229	+ fi
	230	+ cp -p "$src" "$dst" 2> >(grep -v "set flags (was:" >&2)
	231	+ return $?
	232	+}
	233	+
	234	+ensure_unique_destination_path() {
	235	+ local desired_path="$1"
	236	+
	237	+ if [[ -z "$desired_path" ]]; then
	238	+ return 1
	239	+ fi
	240	+
	241	+ if ! destination_path_unavailable "$desired_path"; then
	242	+ echo "$desired_path"
	243	+ return 0
	244	+ fi
	245	+
	246	+ local dir_path filename ext stem candidate
	247	+ dir_path=$(dirname "$desired_path")
	248	+ filename=$(basename "$desired_path")
	249	+
	250	+ if [[ "$filename" == . ]]; then
	251	+ ext="${filename##*.}"
	252	+ stem="${filename%.*}"
	253	+ else
	254	+ ext=""
	255	+ stem="$filename"
	256	+ fi
	257	+
	258	+ local i
	259	+ i=1
	260	+ while [[ $i -le 9999 ]]; do
	261	+ if [[ -n "$ext" ]]; then
	262	+ candidate="$dir_path/${stem}_${i}.${ext}"
	263	+ else
	264	+ candidate="$dir_path/${stem}_${i}"
	265	+ fi
	266	+ if ! destination_path_unavailable "$candidate"; then
	267	+ echo "$candidate"
	268	+ return 0
	269	+ fi
	270	+ i=$((i + 1))
	271	+ done
	272	+
	273	+ return 1
	274	+}
	275	+
222	276	destination_path_reserved() {
223	277	local candidate="$1"
224	278	local reserved_path
	@@ -647,30 +701,28 @@ format_duration() {
647	701	fi
648	702	}
649	703
650		-format_processing_rate() {
651		- local files_count="$1"
652		- local bytes_count="$2"
653		- local elapsed_seconds="$3"
	704	+format_data_rate() {
	705	+ local bytes_count="$1"
	706	+ local elapsed_seconds="$2"
654	707
655		- awk -v files="$files_count" -v bytes="$bytes_count" -v seconds="$elapsed_seconds" '
	708	+ awk -v bytes="$bytes_count" -v seconds="$elapsed_seconds" '
656	709	BEGIN {
657		- if (seconds <= 0 \|\| files <= 0) {
	710	+ if (seconds <= 0 \|\| bytes <= 0) {
658	711	exit
659	712	}
660	713
661		- files_per_second = files / seconds
662		- files_per_minute = files * 60 / seconds
663	714	mb_per_second = bytes / seconds / 1048576
664		-
665		- if (files_per_second >= 1) {
666		- printf "%.2f files/sec, %.2f MB/sec", files_per_second, mb_per_second
667		- } else {
668		- printf "%.2f files/min, %.2f MB/sec", files_per_minute, mb_per_second
669		- }
	715	+ printf "%.2f MB/sec", mb_per_second
670	716	}
671	717	'
672	718	}
673	719
	720	+report_line() {
	721	+ local label="$1"
	722	+ local value="$2"
	723	+ printf " %-24s %s\n" "$label" "$value"
	724	+}
	725	+
674	726	# Function to extract date from file
675	727	extract_file_date() {
676	728	local file="$1"
	@@ -1171,13 +1223,31 @@ show_report() {
1171	1223	print_color "$GREEN" " PROCESSING REPORT"
1172	1224	print_color "$GREEN" "=========================================="
1173	1225	echo ""
1174		-
	1226	+
	1227	+ echo "Files Summary:"
	1228	+ report_line "Total files found:" "$TOTAL_FILES"
	1229	+ report_line "Successfully processed:" "$PROCESSED_FILES"
	1230	+ report_line "Skipped:" "$SKIPPED_FILES"
	1231	+ report_line "Errors:" "$ERROR_FILES"
	1232	+ echo ""
	1233	+
1175	1234	echo "Size Summary:"
1176		- printf " %-22s %s\n" "Total size found:" "$(format_size $TOTAL_SIZE)"
1177		- printf " %-22s %s\n" "Successfully processed:" "$(format_size $PROCESSED_SIZE)"
1178		-
	1235	+ report_line "Total size found:" "$(format_size $TOTAL_SIZE)"
	1236	+ report_line "Successfully processed:" "$(format_size $PROCESSED_SIZE)"
1179	1237	echo ""
1180		-
	1238	+
	1239	+ echo "Time Summary:"
	1240	+ report_line "Time elapsed:" "$(printf "%02d:%02d:%02d" $hours $minutes $seconds)"
	1241	+ if [[ $elapsed_time -gt 0 && $PROCESSED_SIZE -gt 0 ]]; then
	1242	+ local data_rate
	1243	+ data_rate=$(format_data_rate "$PROCESSED_SIZE" "$elapsed_time")
	1244	+ if [[ -n "$data_rate" ]]; then
	1245	+ report_line "Data rate:" "$data_rate"
	1246	+ fi
	1247	+ fi
	1248	+
	1249	+ echo ""
	1250	+
1181	1251	if [[ $DRY_RUN -eq 1 ]]; then
1182	1252	print_color "$YELLOW" "DRY RUN MODE - No files were actually moved/copied"
1183	1253	elif [[ $KEEP_ORIGINALS -eq 1 ]]; then

	@@ -206,19 +206,29 @@ EOF
206	206	cleanup_old_reports() {
207	207	if [[ -d "$TEST_REPORTS_DIR" ]]; then
208	208	# Count total reports
209		- local total_reports=$(find "$TEST_REPORTS_DIR" -name "*.md" \| wc -l)
	209	+ local total_reports
	210	+ total_reports=$(find "$TEST_REPORTS_DIR" -name "*.md" \| wc -l \| tr -d ' ')
210	211
211	212	if [[ $total_reports -gt 10 ]]; then
212	213	print_color "$BLUE" "Cleaning up old test reports (keeping last 10)..."
213	214
214		- # Find and remove oldest reports, keeping the 10 most recent
215		- find "$TEST_REPORTS_DIR" -name "*.md" -type f -printf '%T@ %p\n' \| \
216		- sort -n \| \
217		- head -n -$((10)) \| \
218		- cut -d' ' -f2- \| \
219		- xargs -r rm
220		-
221		- local remaining=$(find "$TEST_REPORTS_DIR" -name "*.md" \| wc -l)
	215	+ local remove_count
	216	+ remove_count=$((total_reports - 10))
	217	+
	218	+ find "$TEST_REPORTS_DIR" -name "*.md" -type f \| while IFS= read -r report_file; do
	219	+ local report_mtime
	220	+ if report_mtime=$(stat -f %m "$report_file" 2>/dev/null); then
	221	+ :
	222	+ else
	223	+ report_mtime=$(stat -c %Y "$report_file" 2>/dev/null \|\| echo 0)
	224	+ fi
	225	+ printf '%s\t%s\n' "$report_mtime" "$report_file"
	226	+ done \| sort -n \| head -n "$remove_count" \| cut -f2- \| while IFS= read -r old_report; do
	227	+ rm -f "$old_report"
	228	+ done
	229	+
	230	+ local remaining
	231	+ remaining=$(find "$TEST_REPORTS_DIR" -name "*.md" \| wc -l \| tr -d ' ')
222	232	print_color "$GREEN" "Kept $remaining most recent test reports"
223	233	fi
224	234	fi