autoNAS / scripts / autonas-media-importer.sh
Newer Older
920 lines | 29.661kb
Bogdan Timofte authored 3 months ago
1
#!/bin/bash
2

            
3
# AutoNAS Media Importer
4
# Advanced media import engine that processes, organizes and imports media files from cameras
Bogdan Timofte authored 2 weeks ago
5
# Usage: autonas-media-importer.sh <source_mount> <destination_path> [options]
6
# Features: organization patterns (ymd/ym/d/h/m/y), GoPro handling, metadata sync
Bogdan Timofte authored 3 months ago
7

            
8
LOG_TAG="autonas-import"
Bogdan Timofte authored 2 weeks ago
9
VERSION="2.0"
10

            
11
# Default values (compatible with original)
12
ORGANIZATION="ymd"
13
FILENAME_MODE="full"
14
VERIFY_MODE="size"
15
DATE_SOURCE="auto"
16
SYNC_METADATA=0
17
UNATTENDED=1
18
COLLECT_UNSORTABLE=0
19
KEEP_EMPTY_DIRS=1
20
DRY_RUN=0
21
VERBOSE=0
22
KEEP_ORIGINALS=0
23
FILE_LIMIT=0
24
CONFLICT_APPLY_ALL=""
25
RESOLVED_DESTINATION_PATH=""
26
RESERVED_DESTINATION_PATHS=()
Bogdan Timofte authored 2 weeks ago
27
WORK_BASE="${AUTONAS_IMPORT_WORKDIR:-/dev/shm/autonas-media-importer}"
Bogdan Timofte authored 2 weeks ago
28

            
29
TOTAL_FILES=0
30
PROCESSED_FILES=0
31
SKIPPED_FILES=0
32
ERROR_FILES=0
33
TOTAL_SIZE=0
34
PROCESSED_SIZE=0
35
START_TIME=$(date +%s)
36

            
37
RED='\033[0;31m'
38
GREEN='\033[0;32m'
39
YELLOW='\033[1;33m'
40
BLUE=$'\033[1;36m'
41
NC='\033[0m'
42

            
43
MEDIA_EXTENSIONS=("*.jpg" "*.jpeg" "*.png" "*.tiff" "*.tif" "*.cr2" "*.nef" "*.arw" "*.dng" "*.raw" "*.mp4" "*.mov" "*.avi" "*.mts" "*.m2ts" "*.mkv" "*.wmv" "*.3gp" "*.m4v")
44

            
45
print_color() {
46
    local color="$1"
47
    local message="$2"
48
    echo -e "${color}${message}${NC}"
49
}
Bogdan Timofte authored 3 months ago
50

            
51
log_message() {
52
    local message="$1"
Bogdan Timofte authored 2 weeks ago
53
    local priority="${2:-info}"
54

            
Bogdan Timofte authored 3 months ago
55
    logger -p "local0.$priority" -t "$LOG_TAG" "$message"
Bogdan Timofte authored 2 weeks ago
56

            
Bogdan Timofte authored 3 months ago
57
    if [ -t 1 ]; then
Bogdan Timofte authored 2 weeks ago
58
        case "$priority" in
59
            "err")
60
                print_color "$RED" "$(date '+%Y-%m-%d %H:%M:%S') - ERROR: $message" >&2
61
                ;;
62
            "warning")
63
                print_color "$YELLOW" "$(date '+%Y-%m-%d %H:%M:%S') - WARNING: $message"
64
                ;;
65
            "info")
66
                if [[ $VERBOSE -eq 1 ]]; then
67
                    print_color "$BLUE" "$(date '+%Y-%m-%d %H:%M:%S') - INFO: $message"
68
                fi
69
                ;;
70
            *)
71
                echo "$(date '+%Y-%m-%d %H:%M:%S') - $message"
72
                ;;
73
        esac
Bogdan Timofte authored 3 months ago
74
    fi
75
}
76

            
Bogdan Timofte authored 2 weeks ago
77
show_help() {
78
    cat << EOF
79
AutoNAS Media Importer v$VERSION
80
Advanced camera import engine
81

            
82
Usage:
83
    $0 <source_mount> <destination_path> [OPTIONS]
84

            
85
Arguments:
86
    source_mount      Mount point of camera
87
    destination_path  Destination directory for imported files
88

            
89
Options:
90
    -o, --organization   y|m|d|h|ym|ymd (default: ymd)
91
    -F, --filename-mode  auto|full|orig (default: full)
92
    --date-source        auto|exif|filesystem (default: auto)
93
    --sync-metadata      Write date into destination metadata
Bogdan Timofte authored 2 weeks ago
94
    --unattended         Never prompt; resolve destination conflicts with numeric suffixes
Bogdan Timofte authored 2 weeks ago
95
    --collect-unsortable Put undated files into DEST/unsortable
96
    --keep-empty-dirs    Keep empty directories after processing
97
    --dry-run            Show actions without changing files
98
    --keep-originals     Copy files instead of moving
99
    --verbose            Enable verbose output
100
    --limit N            Process only N files
101
    -h, --help           Show this help
102
EOF
103
}
104

            
105
check_dependencies() {
106
    if ! command -v exiftool &> /dev/null; then
107
        print_color "$RED" "ERROR: exiftool is required but not installed"
108
        exit 1
109
    fi
110
}
111

            
112
# Utility functions
113
get_file_size() {
114
    local file="$1"
115
    if [[ -f "$file" ]]; then
116
        if stat -c%s "$file" >/dev/null 2>&1; then
117
            stat -c%s "$file" 2>/dev/null
118
        elif stat -f%z "$file" >/dev/null 2>&1; then
119
            stat -f%z "$file" 2>/dev/null
120
        else
121
            ls -ln "$file" | awk '{print $5}'
122
        fi
123
    else
124
        echo "0"
125
    fi
126
}
127

            
128
is_gopro_media_file() {
129
    local filename
130
    filename=$(basename "$1")
131
    [[ "$filename" =~ ^G[HPX][0-9]{6}\.[Mm][Pp]4$ ]]
132
}
133

            
134
should_prefer_gopro_filesystem_date() {
135
    is_gopro_media_file "$1"
136
}
137

            
Bogdan Timofte authored 2 weeks ago
138
date_to_exiftool_format() {
139
    local date_str="$1"
140

            
141
    if [[ "$date_str" =~ ^([0-9]{4})-([0-9]{2})-([0-9]{2})[[:space:]]+([0-9]{2}):([0-9]{2}):([0-9]{2})$ ]]; then
142
        echo "${BASH_REMATCH[1]}:${BASH_REMATCH[2]}:${BASH_REMATCH[3]} ${BASH_REMATCH[4]}:${BASH_REMATCH[5]}:${BASH_REMATCH[6]}"
143
        return 0
144
    fi
145

            
146
    return 1
147
}
148

            
149
sync_destination_metadata_to_date() {
Bogdan Timofte authored 2 weeks ago
150
    local file="$1"
Bogdan Timofte authored 2 weeks ago
151
    local date_str="$2"
152
    local exif_date
153

            
154
    exif_date=$(date_to_exiftool_format "$date_str") || return 1
155

            
156
    # GoPro filesystem timestamps are local camera time. QuickTime integer
157
    # timestamps should be stored as UTC, so let ExifTool convert from local
158
    # time on write instead of storing the local clock value as if it were UTC.
159
    exiftool -api QuickTimeUTC=1 -overwrite_original \
160
        "-CreateDate=$exif_date" \
161
        "-DateTimeOriginal=$exif_date" \
162
        "-DateTime=$exif_date" \
163
        "-ModifyDate=$exif_date" \
164
        "-MediaCreateDate=$exif_date" \
165
        "-TrackCreateDate=$exif_date" \
166
        "-QuickTime:CreateDate=$exif_date" \
167
        "-QuickTime:ModifyDate=$exif_date" \
168
        "$file" >/dev/null 2>&1
169
}
170

            
171
set_filesystem_mtime_to_date() {
172
    local file="$1"
173
    local date_str="$2"
174

            
175
    if [[ "$OSTYPE" == "darwin"* ]]; then
176
        local touch_date
177
        touch_date=$(date -j -f "%Y-%m-%d %H:%M:%S" "$date_str" "+%Y%m%d%H%M.%S" 2>/dev/null) || return 1
178
        touch -t "$touch_date" "$file"
179
    else
180
        touch -d "$date_str" "$file"
181
    fi
182
}
183

            
184
should_sync_imported_metadata() {
185
    local original_filename="$1"
186
    local date_source="$2"
187

            
188
    if [[ $SYNC_METADATA -eq 1 ]]; then
189
        return 0
Bogdan Timofte authored 2 weeks ago
190
    fi
191

            
Bogdan Timofte authored 2 weeks ago
192
    [[ "$date_source" == Filesystem* ]] && is_gopro_media_file "$original_filename"
193
}
194

            
195
filesystem_date_reference() {
196
    local file="$1"
197

            
198
    # GoPro MP4 files carry the capture time reliably in the file's own
199
    # filesystem timestamp. THM/LRV sidecars can drift to chapter/end times on
200
    # some cards, so they must not override the MP4 timestamp.
Bogdan Timofte authored 2 weeks ago
201
    echo "$file"
202
}
203

            
204
extract_filesystem_date() {
205
    local file="$1"
206
    if [[ ! -e "$file" ]]; then
207
        return 2
208
    fi
209

            
210
    local epoch=""
211
    if [[ "$OSTYPE" == "darwin"* ]]; then
212
        epoch=$(stat -f %m "$file" 2>/dev/null || echo "")
213
        [[ -n "$epoch" ]] || return 2
214
        date -j -r "$epoch" "+%Y-%m-%d %H:%M:%S" 2>/dev/null || return 2
215
    else
216
        epoch=$(stat -c %Y "$file" 2>/dev/null || echo "")
217
        [[ -n "$epoch" ]] || return 2
218
        date -d "@$epoch" "+%Y-%m-%d %H:%M:%S" 2>/dev/null || return 2
219
    fi
220
}
221

            
222
destination_path_unavailable() {
223
    local candidate="$1"
224
    [[ -e "$candidate" ]] && return 0
225
    for reserved_path in "${RESERVED_DESTINATION_PATHS[@]}"; do
226
        [[ "$reserved_path" == "$candidate" ]] && return 0
227
    done
228
    return 1
229
}
230

            
231
reserve_destination_path() {
232
    local candidate="$1"
233
    if [[ -n "$candidate" ]]; then
234
        RESERVED_DESTINATION_PATHS+=("$candidate")
235
    fi
236
}
237

            
238
extract_file_date() {
239
    local file="$1"
240
    local create_date=""
241
    local date_source=""
242

            
243
    if [[ "$DATE_SOURCE" == "filesystem" ]] || { [[ "$DATE_SOURCE" == "auto" ]] && should_prefer_gopro_filesystem_date "$file"; }; then
244
        local filesystem_reference
245
        filesystem_reference=$(filesystem_date_reference "$file")
246
        create_date=$(extract_filesystem_date "$filesystem_reference") || return 2
247
        if is_gopro_media_file "$file"; then
248
            echo "$create_date|Filesystem:GoPro"
249
        else
250
            echo "$create_date|Filesystem"
251
        fi
252
        return 0
253
    fi
254

            
255
    # Try EXIF data
256
    local exif_output=$(exiftool -G1 -s -CreateDate -DateTimeOriginal -DateTime "$file" 2>/dev/null)
257
    if [[ -n "$exif_output" ]]; then
258
        while IFS= read -r line; do
259
            if [[ "$line" =~ ^\[([^\]]+)\][[:space:]]*([^:]+)[[:space:]]*:[[:space:]]*(.+)$ ]]; then
260
                local group="${BASH_REMATCH[1]}"
261
                local tag="${BASH_REMATCH[2]}"
262
                local value="${BASH_REMATCH[3]}"
263
                tag=$(echo "$tag" | sed 's/[[:space:]]*$//')
264

            
265
                if [[ "$tag" == "DateTimeOriginal" && -z "$create_date" ]]; then
266
                    create_date="$value"
267
                    date_source="$group:$tag"
268
                elif [[ "$tag" == "CreateDate" && "$date_source" != *"DateTimeOriginal"* ]]; then
269
                    create_date="$value"
270
                    date_source="$group:$tag"
271
                elif [[ "$tag" == "DateTime" && -z "$create_date" ]]; then
272
                    create_date="$value"
273
                    date_source="$group:$tag"
274
                fi
275
            fi
276
        done <<< "$exif_output"
277
    fi
278

            
279
    # Fallback to filesystem in auto mode
280
    if [[ -z "$create_date" && "$DATE_SOURCE" == "auto" ]]; then
281
        local filesystem_reference
282
        filesystem_reference=$(filesystem_date_reference "$file")
283
        create_date=$(extract_filesystem_date "$filesystem_reference") || return 2
284
        date_source="Filesystem"
285
    fi
286

            
287
    [[ -z "$create_date" ]] && return 2
288

            
289
    # Convert EXIF format (YYYY:MM:DD HH:MM:SS) to standard
290
    if [[ "$create_date" =~ ^([0-9]{4}):([0-9]{1,2}):([0-9]{1,2})[[:space:]]*([0-9]{1,2}):([0-9]{1,2}):([0-9]{1,2})$ ]]; then
291
        local year="${BASH_REMATCH[1]}"
292
        local month=$(printf "%02d" "$((10#${BASH_REMATCH[2]}))")
293
        local day=$(printf "%02d" "$((10#${BASH_REMATCH[3]}))")
294
        local hour=$(printf "%02d" "$((10#${BASH_REMATCH[4]}))")
295
        local minute=$(printf "%02d" "$((10#${BASH_REMATCH[5]}))")
296
        local second=$(printf "%02d" "$((10#${BASH_REMATCH[6]}))")
297
        create_date="$year-$month-$day $hour:$minute:$second"
298
    fi
299

            
300
    # QuickTime UTC conversion
301
    if [[ "$date_source" == *"QuickTime"* ]]; then
302
        if [[ "$OSTYPE" == "darwin"* ]]; then
303
            local utc_timestamp=$(TZ=UTC date -j -f "%Y-%m-%d %H:%M:%S" "$create_date" "+%s" 2>/dev/null)
304
            [[ -n "$utc_timestamp" ]] && create_date=$(date -j -r "$utc_timestamp" "+%Y-%m-%d %H:%M:%S" 2>/dev/null)
305
        else
306
            local utc_timestamp=$(date -d "$create_date UTC" "+%s" 2>/dev/null)
307
            [[ -n "$utc_timestamp" ]] && create_date=$(date -d "@$utc_timestamp" "+%Y-%m-%d %H:%M:%S" 2>/dev/null)
308
        fi
309
    fi
310

            
311
    echo "$create_date|$date_source"
312
    return 0
313
}
314

            
315
generate_destination_path() {
316
    local date_str="$1"
317
    local original_filename="$2"
318
    local base_destination="$3"
319

            
320
    local year month day hour minute second
321
    if [[ "$OSTYPE" == "darwin"* ]]; then
322
        if [[ "$date_str" =~ ^([0-9]{4})-([0-9]{1,2})-([0-9]{1,2})[[:space:]]*([0-9]{1,2}):([0-9]{1,2}):([0-9]{1,2})$ ]]; then
323
            year="${BASH_REMATCH[1]}"
324
            month=$(printf "%02d" "$((10#${BASH_REMATCH[2]}))")
325
            day=$(printf "%02d" "$((10#${BASH_REMATCH[3]}))")
326
            hour=$(printf "%02d" "$((10#${BASH_REMATCH[4]}))")
327
            minute=$(printf "%02d" "$((10#${BASH_REMATCH[5]}))")
328
            second=$(printf "%02d" "$((10#${BASH_REMATCH[6]}))")
329
        else
330
            return 1
331
        fi
332
    else
333
        year=$(date -d "$date_str" "+%Y" 2>/dev/null)
334
        month=$(date -d "$date_str" "+%m" 2>/dev/null)
335
        day=$(date -d "$date_str" "+%d" 2>/dev/null)
336
        hour=$(date -d "$date_str" "+%H" 2>/dev/null)
337
        minute=$(date -d "$date_str" "+%M" 2>/dev/null)
338
        second=$(date -d "$date_str" "+%S" 2>/dev/null)
339
    fi
340

            
341
    [[ -z "$year" || -z "$month" || -z "$day" ]] && return 1
342

            
343
    local extension="${original_filename##*.}"
344
    local lowercase_ext=$(echo "$extension" | tr '[:upper:]' '[:lower:]')
345

            
346
    local dir_path=""
347
    local filename=""
348

            
349
    case "$ORGANIZATION" in
350
        "y")
351
            dir_path="$base_destination/$year"
352
            filename="${month}-${day}_${hour}-${minute}-${second}.${lowercase_ext}"
353
            ;;
354
        "m")
355
            dir_path="$base_destination/$year/$month"
356
            filename="${day}_${hour}-${minute}-${second}.${lowercase_ext}"
357
            ;;
358
        "d")
359
            dir_path="$base_destination/$year/$month/$day"
360
            filename="${hour}-${minute}-${second}.${lowercase_ext}"
361
            ;;
362
        "h")
363
            dir_path="$base_destination/$year/$month/$day/$hour"
364
            filename="${minute}-${second}.${lowercase_ext}"
365
            ;;
366
        "ym")
367
            dir_path="$base_destination/${year}-${month}"
368
            filename="${day}_${hour}-${minute}-${second}.${lowercase_ext}"
369
            ;;
370
        "ymd")
371
            dir_path="$base_destination/${year}-${month}-${day}"
372
            filename="${hour}-${minute}-${second}.${lowercase_ext}"
373
            ;;
374
        *)
375
            return 1
376
            ;;
377
    esac
378

            
379
    case "$FILENAME_MODE" in
380
        orig)
381
            filename="$original_filename"
382
            ;;
383
        full)
384
            filename="${year}-${month}-${day}_${hour}-${minute}-${second}.${lowercase_ext}"
385
            ;;
386
        auto)
387
            ;;
388
        *)
389
            filename="${year}-${month}-${day}_${hour}-${minute}-${second}.${lowercase_ext}"
390
            ;;
391
    esac
392

            
393
    echo "$dir_path/$filename"
394
    return 0
395
}
396

            
397
ensure_unique_destination_path() {
398
    local desired_path="$1"
399
    local counter=1
400
    local resolved_path="$desired_path"
401

            
402
    while destination_path_unavailable "$resolved_path"; do
403
        local dir=$(dirname "$desired_path")
404
        local base=$(basename "$desired_path")
405
        local name_without_ext="${base%.*}"
406
        local ext="${base##*.}"
407

            
408
        if [[ "$ext" == "$base" ]]; then
409
            resolved_path="$dir/${name_without_ext}_${counter}"
410
        else
411
            resolved_path="$dir/${name_without_ext}_${counter}.$ext"
412
        fi
413
        counter=$((counter + 1))
414

            
415
        [[ $counter -gt 1000 ]] && return 1
416
    done
417

            
418
    echo "$resolved_path"
419
    return 0
420
}
421

            
422
resolve_destination_conflict() {
423
    local desired_path="$1"
424
    RESOLVED_DESTINATION_PATH=""
425

            
426
    [[ -z "$desired_path" ]] && return 1
427

            
428
    if ! destination_path_unavailable "$desired_path"; then
429
        RESOLVED_DESTINATION_PATH="$desired_path"
430
        reserve_destination_path "$RESOLVED_DESTINATION_PATH"
431
        return 0
432
    fi
433

            
434
    local resolved_path
435
    resolved_path=$(ensure_unique_destination_path "$desired_path")
436
    [[ -z "$resolved_path" ]] && return 1
437

            
438
    RESOLVED_DESTINATION_PATH="$resolved_path"
439
    reserve_destination_path "$RESOLVED_DESTINATION_PATH"
440
    return 0
441
}
442

            
443
safe_mv() {
444
    local src="$1" dst="$2"
445
    mv "$src" "$dst" 2> >(grep -v "set flags (was:" >&2)
446
}
447

            
Bogdan Timofte authored 2 weeks ago
448
safe_cp() {
449
    local src="$1" dst="$2"
450
    cp -p "$src" "$dst" 2> >(grep -v "set flags (was:" >&2)
451
}
452

            
453
verify_copied_file() {
454
    local src="$1"
455
    local dst="$2"
456

            
457
    if [[ ! -f "$dst" ]]; then
458
        log_message "Verified copy missing at destination: $dst" "err"
459
        return 1
460
    fi
461

            
462
    if [[ "$VERIFY_MODE" == "none" ]]; then
463
        return 0
464
    fi
465

            
466
    local src_size dst_size
467
    src_size=$(get_file_size "$src")
468
    dst_size=$(get_file_size "$dst")
469

            
470
    if [[ "$src_size" != "$dst_size" ]]; then
471
        log_message "Size mismatch after copy: $src ($src_size) != $dst ($dst_size)" "err"
472
        return 1
473
    fi
474

            
475
    if [[ "$VERIFY_MODE" == "strict" ]] && ! cmp -s "$src" "$dst"; then
476
        log_message "Content mismatch after copy: $src -> $dst" "err"
477
        return 1
478
    fi
479

            
480
    return 0
481
}
482

            
483
copy_with_verification() {
484
    local src="$1"
485
    local dst="$2"
486

            
487
    if [[ -e "$dst" ]]; then
488
        log_message "Refusing to overwrite existing destination: $dst" "err"
489
        return 1
490
    fi
491

            
492
    local dst_dir tmp
493
    dst_dir=$(dirname "$dst")
494
    tmp=$(mktemp "$dst_dir/.media-importer.$(basename "$dst").tmp.XXXXXX") || return 1
495
    rm -f "$tmp"
496

            
497
    if ! safe_cp "$src" "$tmp"; then
498
        rm -f "$tmp"
499
        return 1
500
    fi
501

            
502
    if ! verify_copied_file "$src" "$tmp"; then
503
        rm -f "$tmp"
504
        return 1
505
    fi
506

            
507
    if [[ -e "$dst" ]]; then
508
        log_message "Destination appeared during copy, refusing to overwrite: $dst" "err"
509
        rm -f "$tmp"
510
        return 1
511
    fi
512

            
513
    if ! safe_mv "$tmp" "$dst"; then
514
        rm -f "$tmp"
515
        return 1
516
    fi
517
}
518

            
519
available_bytes_for_path() {
520
    local path="$1"
521

            
522
    df -PB1 "$path" 2>/dev/null | awk 'NR==2 {print $4}'
523
}
524

            
525
staging_directory_for_file() {
526
    local file_size="$1"
527
    local needs_metadata_sync="$2"
528
    local dest_dir="$3"
529
    local required_bytes available_bytes
530

            
531
    required_bytes=$((file_size + 67108864))
532
    if [[ "$needs_metadata_sync" -eq 1 ]]; then
533
        required_bytes=$((file_size * 2 + 67108864))
534
    fi
535

            
536
    if mkdir -p "$WORK_BASE" 2>/dev/null; then
537
        available_bytes=$(available_bytes_for_path "$WORK_BASE")
538
        if [[ "$available_bytes" =~ ^[0-9]+$ && "$available_bytes" -gt "$required_bytes" ]]; then
539
            echo "$WORK_BASE"
540
            return 0
541
        fi
542
        log_message "Not enough space in $WORK_BASE for staging; falling back to destination staging" "warning"
543
    fi
544

            
545
    echo "$dest_dir"
546
}
547

            
548
copy_with_staging() {
549
    local src="$1"
550
    local dst="$2"
551
    local date_str="$3"
552
    local sync_metadata="$4"
553

            
554
    if [[ -e "$dst" ]]; then
555
        log_message "Refusing to overwrite existing destination: $dst" "err"
556
        return 1
557
    fi
558

            
559
    local dst_dir src_size stage_dir stage_tmp
560
    dst_dir=$(dirname "$dst")
561
    src_size=$(get_file_size "$src")
562
    stage_dir=$(staging_directory_for_file "$src_size" "$sync_metadata" "$dst_dir")
563
    mkdir -p "$stage_dir" || return 1
564

            
565
    stage_tmp=$(mktemp "$stage_dir/.media-importer.$(basename "$dst").stage.XXXXXX") || return 1
566
    rm -f "$stage_tmp"
567

            
568
    if ! safe_cp "$src" "$stage_tmp"; then
569
        rm -f "$stage_tmp"
570
        return 1
571
    fi
572

            
573
    if ! verify_copied_file "$src" "$stage_tmp"; then
574
        rm -f "$stage_tmp"
575
        return 1
576
    fi
577

            
578
    if [[ "$sync_metadata" -eq 1 ]]; then
579
        if ! sync_destination_metadata_to_date "$stage_tmp" "$date_str"; then
580
            log_message "Failed to sync metadata date for staged file: $stage_tmp" "warning"
581
            rm -f "$stage_tmp"
582
            return 1
583
        fi
584
    fi
585

            
586
    set_filesystem_mtime_to_date "$stage_tmp" "$date_str" || log_message "Failed to set filesystem timestamp for staged file: $stage_tmp" "warning"
587

            
588
    if ! copy_with_verification "$stage_tmp" "$dst"; then
589
        rm -f "$stage_tmp"
590
        return 1
591
    fi
592

            
593
    set_filesystem_mtime_to_date "$dst" "$date_str" || log_message "Failed to set filesystem timestamp for $dst" "warning"
594
    rm -f "$stage_tmp"
595
}
596

            
597
verified_move_file() {
598
    local src="$1"
599
    local dst="$2"
600

            
601
    if ! copy_with_verification "$src" "$dst"; then
602
        return 1
603
    fi
604

            
605
    rm -f "$src"
606
}
607

            
Bogdan Timofte authored 2 weeks ago
608
# Process a single file
609
process_file() {
610
    local file="$1"
611
    local relative_path="${file#$SOURCE_MOUNT/}"
612
    local file_size=$(get_file_size "$file")
613
    TOTAL_SIZE=$((TOTAL_SIZE + file_size))
614

            
615
    # Check mount point
616
    if ! mountpoint -q "$SOURCE_MOUNT" 2>/dev/null; then
617
        log_message "Error: Source mount point is no longer available" "err"
618
        return 1
619
    fi
620

            
621
    # Check if file still exists
622
    if [[ ! -f "$file" ]]; then
623
        log_message "Error: File no longer exists: $relative_path" "err"
624
        log_message "Camera appears to be disconnected, stopping import" "warning"
625
        exit 1
626
    fi
627

            
628
    [[ $VERBOSE -eq 1 ]] && log_message "Processing: $relative_path" "info"
629

            
630
    # Extract date
631
    local date_info=$(extract_file_date "$file")
632
    local extract_status=$?
633

            
634
    if [[ $extract_status -eq 2 ]]; then
635
        if [[ $COLLECT_UNSORTABLE -eq 1 ]]; then
636
            local unsortable_dir="$DESTINATION/unsortable"
637
            mkdir -p "$unsortable_dir"
638
            local unsortable_path="$unsortable_dir/$(basename "$file")"
639
            resolve_destination_conflict "$unsortable_path"
640
            [[ $? -eq 0 ]] && unsortable_path="$RESOLVED_DESTINATION_PATH"
641
            if [[ $DRY_RUN -eq 0 ]]; then
642
                safe_mv "$file" "$unsortable_path"
643
                log_message "Moved to unsortable: $relative_path" "info"
644
            fi
645
        else
646
            log_message "No date found for $relative_path - skipping" "warning"
647
        fi
648
        SKIPPED_FILES=$((SKIPPED_FILES + 1))
649
        return 1
650
    elif [[ $extract_status -ne 0 ]]; then
651
        log_message "Failed to extract date from $relative_path" "warning"
652
        SKIPPED_FILES=$((SKIPPED_FILES + 1))
653
        return 1
654
    fi
655

            
656
    local date_str="${date_info%|*}"
657
    local date_source="${date_info#*|}"
658
    [[ $VERBOSE -eq 1 ]] && log_message "Date: $date_str (from $date_source)" "info"
659

            
660
    # Generate destination path
661
    local original_basename=$(basename "$file")
662
    local dest_path=$(generate_destination_path "$date_str" "$original_basename" "$DESTINATION")
663
    [[ $? -ne 0 ]] && { log_message "Could not generate destination path for $relative_path" "err"; ERROR_FILES=$((ERROR_FILES + 1)); return 1; }
664

            
665
    local desired_dest_path="$dest_path"
666
    resolve_destination_conflict "$dest_path"
667
    [[ $? -eq 0 ]] && dest_path="$RESOLVED_DESTINATION_PATH" || { ERROR_FILES=$((ERROR_FILES + 1)); return 1; }
668

            
669
    [[ "$dest_path" != "$desired_dest_path" ]] && log_message "Destination conflict resolved" "info"
670

            
671
    local dest_dir=$(dirname "$dest_path")
672

            
673
    if [[ $DRY_RUN -eq 1 ]]; then
674
        if [[ $KEEP_ORIGINALS -eq 1 ]]; then
675
            echo "Would copy: $relative_path -> ${dest_path#$DESTINATION/}"
676
        else
677
            echo "Would move: $relative_path -> ${dest_path#$DESTINATION/}"
678
        fi
Bogdan Timofte authored 2 weeks ago
679
        if should_sync_imported_metadata "$original_basename" "$date_source"; then
680
            echo "Would sync metadata date: ${dest_path#$DESTINATION/} -> $date_str"
681
        fi
Bogdan Timofte authored 2 weeks ago
682
        PROCESSED_FILES=$((PROCESSED_FILES + 1))
683
        PROCESSED_SIZE=$((PROCESSED_SIZE + file_size))
684
        return 0
685
    fi
686

            
687
    mkdir -p "$dest_dir" || { log_message "Could not create directory: $dest_dir" "err"; ERROR_FILES=$((ERROR_FILES + 1)); return 1; }
688

            
Bogdan Timofte authored 2 weeks ago
689
    local sync_metadata_after_copy=0
690
    if should_sync_imported_metadata "$original_basename" "$date_source"; then
691
        sync_metadata_after_copy=1
692
    fi
693

            
Bogdan Timofte authored 2 weeks ago
694
    if [[ $KEEP_ORIGINALS -eq 1 ]]; then
Bogdan Timofte authored 2 weeks ago
695
        if copy_with_staging "$file" "$dest_path" "$date_str" "$sync_metadata_after_copy"; then
Bogdan Timofte authored 2 weeks ago
696
            log_message "Copied: $relative_path" "info"
697
            [[ $VERBOSE -eq 1 ]] && echo "✓ Copied"
698
            PROCESSED_FILES=$((PROCESSED_FILES + 1))
699
            PROCESSED_SIZE=$((PROCESSED_SIZE + file_size))
700
            return 0
701
        else
702
            log_message "Failed to copy $relative_path" "err"
703
            ERROR_FILES=$((ERROR_FILES + 1))
704
            return 1
705
        fi
706
    else
Bogdan Timofte authored 2 weeks ago
707
        if copy_with_staging "$file" "$dest_path" "$date_str" "$sync_metadata_after_copy" && rm -f "$file"; then
Bogdan Timofte authored 2 weeks ago
708
            log_message "Moved: $relative_path" "info"
709
            [[ $VERBOSE -eq 1 ]] && echo "✓ Moved"
710
            PROCESSED_FILES=$((PROCESSED_FILES + 1))
711
            PROCESSED_SIZE=$((PROCESSED_SIZE + file_size))
712
            return 0
713
        else
714
            log_message "Failed to move $relative_path" "err"
715
            ERROR_FILES=$((ERROR_FILES + 1))
716
            return 1
717
        fi
718
    fi
Bogdan Timofte authored 3 months ago
719
}
720

            
721
# Parse command line arguments
722
SOURCE_MOUNT=""
723
DESTINATION=""
724

            
725
while [[ $# -gt 0 ]]; do
726
    case $1 in
Bogdan Timofte authored 2 weeks ago
727
        -o|--organization)
728
            ORGANIZATION="$2"
729
            [[ ! "$ORGANIZATION" =~ ^(y|m|d|h|ym|ymd)$ ]] && { echo "Invalid organization pattern"; exit 1; }
730
            shift 2
731
            ;;
732
        -F|--filename-mode)
733
            FILENAME_MODE="$2"
734
            [[ ! "$FILENAME_MODE" =~ ^(auto|full|orig)$ ]] && { echo "Invalid filename mode"; exit 1; }
735
            shift 2
736
            ;;
737
        --date-source)
738
            DATE_SOURCE="$2"
739
            [[ ! "$DATE_SOURCE" =~ ^(auto|exif|filesystem)$ ]] && { echo "Invalid date source"; exit 1; }
740
            shift 2
741
            ;;
742
        --sync-metadata)
743
            SYNC_METADATA=1
744
            shift
745
            ;;
Bogdan Timofte authored 2 weeks ago
746
        --unattended)
747
            UNATTENDED=1
748
            shift
749
            ;;
Bogdan Timofte authored 2 weeks ago
750
        --collect-unsortable)
751
            COLLECT_UNSORTABLE=1
752
            shift
753
            ;;
754
        --keep-empty-dirs)
755
            KEEP_EMPTY_DIRS=0
756
            shift
757
            ;;
Bogdan Timofte authored 3 months ago
758
        --dry-run)
759
            DRY_RUN=1
760
            shift
761
            ;;
762
        --keep-originals)
763
            KEEP_ORIGINALS=1
764
            shift
765
            ;;
Bogdan Timofte authored 2 weeks ago
766
        -v|--verbose)
Bogdan Timofte authored 3 months ago
767
            VERBOSE=1
768
            shift
769
            ;;
770
        --limit)
771
            FILE_LIMIT="$2"
Bogdan Timofte authored 2 weeks ago
772
            [[ ! "$FILE_LIMIT" =~ ^[0-9]+$ ]] && { echo "Error: --limit requires a number"; exit 1; }
Bogdan Timofte authored 3 months ago
773
            shift 2
774
            ;;
Bogdan Timofte authored 2 weeks ago
775
        -h|--help)
776
            show_help
Bogdan Timofte authored 3 months ago
777
            exit 0
778
            ;;
779
        -*)
780
            echo "Unknown option: $1"
781
            exit 1
782
            ;;
783
        *)
784
            if [[ -z "$SOURCE_MOUNT" ]]; then
785
                SOURCE_MOUNT="$1"
786
            elif [[ -z "$DESTINATION" ]]; then
787
                DESTINATION="$1"
788
            else
789
                echo "Too many arguments"
790
                exit 1
791
            fi
792
            shift
793
            ;;
794
    esac
795
done
796

            
Bogdan Timofte authored 2 weeks ago
797
[[ -z "$SOURCE_MOUNT" || -z "$DESTINATION" ]] && { echo "Error: Both source_mount and destination_path are required"; show_help; exit 1; }
Bogdan Timofte authored 3 months ago
798

            
Bogdan Timofte authored 2 weeks ago
799
# Validate paths
800
[[ ! -d "$SOURCE_MOUNT" ]] && { log_message "Error: Source mount point does not exist: $SOURCE_MOUNT" "err"; exit 1; }
801
! mountpoint -q "$SOURCE_MOUNT" 2>/dev/null && log_message "Warning: Source path is not a mount point: $SOURCE_MOUNT" "warning"
Bogdan Timofte authored 3 months ago
802

            
803
if [[ $DRY_RUN -eq 0 ]]; then
Bogdan Timofte authored 2 weeks ago
804
    mkdir -p "$DESTINATION" || { log_message "Error: Cannot create destination directory: $DESTINATION" "err"; exit 1; }
Bogdan Timofte authored 3 months ago
805
fi
806

            
Bogdan Timofte authored 2 weeks ago
807
check_dependencies
Bogdan Timofte authored 3 months ago
808

            
Bogdan Timofte authored 2 weeks ago
809
# Find camera directories
Bogdan Timofte authored 3 months ago
810
find_camera_directories() {
811
    local search_patterns=("DCIM" "PRIVATE" "MP_ROOT" "AVCHD" "Photos" "Videos")
812
    local found_dirs=()
Bogdan Timofte authored 2 weeks ago
813

            
814
    ! timeout 3 ls "$SOURCE_MOUNT" >/dev/null 2>&1 && { log_message "Error: Mount point is not accessible" "err"; exit 1; }
815

            
Bogdan Timofte authored 3 months ago
816
    for pattern in "${search_patterns[@]}"; do
817
        while IFS= read -r -d '' dir; do
818
            found_dirs+=("$dir")
819
        done < <(timeout 5 find "$SOURCE_MOUNT" -maxdepth 3 -type d -iname "$pattern" -print0 2>/dev/null)
820
    done
Bogdan Timofte authored 2 weeks ago
821

            
Bogdan Timofte authored 3 months ago
822
    if [[ ${#found_dirs[@]} -eq 0 ]]; then
823
        log_message "No camera directories found, searching for media files..." "info"
Bogdan Timofte authored 2 weeks ago
824
        for ext in "${MEDIA_EXTENSIONS[@]}"; do
Bogdan Timofte authored 3 months ago
825
            while IFS= read -r -d '' file; do
826
                local dir=$(dirname "$file")
Bogdan Timofte authored 2 weeks ago
827
                local already_added=0
828
                for found_dir in "${found_dirs[@]}"; do
829
                    [[ "$found_dir" == "$dir" ]] && { already_added=1; break; }
830
                done
831
                [[ $already_added -eq 0 ]] && found_dirs+=("$dir")
Bogdan Timofte authored 3 months ago
832
            done < <(timeout 5 find "$SOURCE_MOUNT" -maxdepth 3 -type f -iname "$ext" -print0 2>/dev/null)
833
        done
834
    fi
Bogdan Timofte authored 2 weeks ago
835

            
Bogdan Timofte authored 3 months ago
836
    printf '%s\n' "${found_dirs[@]}" | sort -u
837
}
838

            
839
log_message "Starting camera import from $SOURCE_MOUNT to $DESTINATION" "info"
Bogdan Timofte authored 2 weeks ago
840
[[ $DRY_RUN -eq 1 ]] && log_message "DRY RUN MODE - No files will be moved/copied" "info"
841
[[ $KEEP_ORIGINALS -eq 1 ]] && log_message "KEEP ORIGINALS MODE - Files will be copied instead of moved" "info"
Bogdan Timofte authored 3 months ago
842

            
843
log_message "Scanning for camera directories..." "info"
844
camera_dirs=$(find_camera_directories)
845

            
846
if [[ -z "$camera_dirs" ]]; then
847
    log_message "No camera directories or media files found in $SOURCE_MOUNT" "warning"
848
    exit 0
849
fi
850

            
851
echo "Found camera directories:"
852
echo "$camera_dirs" | while IFS= read -r dir; do
853
    echo "  $dir"
854
done
855

            
Bogdan Timofte authored 2 weeks ago
856
# Clean up GLV files (preview files - usually not needed)
Bogdan Timofte authored 3 months ago
857
log_message "Cleaning up GLV preview files..." "info"
858
glv_count=0
859
while IFS= read -r dir; do
860
    if [[ $DRY_RUN -eq 1 ]]; then
861
        glv_files=$(find "$dir" -type f -iname "*.glv" 2>/dev/null | wc -l)
Bogdan Timofte authored 2 weeks ago
862
        [[ $glv_files -gt 0 ]] && echo "Would delete $glv_files GLV files from $dir" && glv_count=$((glv_count + glv_files))
Bogdan Timofte authored 3 months ago
863
    else
864
        while IFS= read -r -d '' glv_file; do
Bogdan Timofte authored 2 weeks ago
865
            rm "$glv_file" 2>/dev/null && glv_count=$((glv_count + 1))
Bogdan Timofte authored 3 months ago
866
        done < <(find "$dir" -type f -iname "*.glv" -print0 2>/dev/null)
867
    fi
868
done <<< "$camera_dirs"
869

            
Bogdan Timofte authored 2 weeks ago
870
[[ $glv_count -gt 0 ]] && log_message "GLV files cleaned up: $glv_count" "info"
Bogdan Timofte authored 3 months ago
871

            
872
# Process media files
873
log_message "Processing media files..." "info"
874

            
875
max_files_to_show=20
876
files_shown=0
877
files_processed_count=0
878

            
879
while IFS= read -r dir; do
Bogdan Timofte authored 2 weeks ago
880
    ! mountpoint -q "$SOURCE_MOUNT" 2>/dev/null && { log_message "Camera disconnected, stopping import" "warning"; break; }
881

            
882
    for ext in "${MEDIA_EXTENSIONS[@]}"; do
Bogdan Timofte authored 3 months ago
883
        while IFS= read -r -d '' file; do
Bogdan Timofte authored 2 weeks ago
884
            TOTAL_FILES=$((TOTAL_FILES + 1))
Bogdan Timofte authored 3 months ago
885
            files_processed_count=$((files_processed_count + 1))
Bogdan Timofte authored 2 weeks ago
886

            
887
            ! mountpoint -q "$SOURCE_MOUNT" 2>/dev/null && { log_message "Camera disconnected during processing" "warning"; exit 1; }
888

            
889
            [[ $FILE_LIMIT -gt 0 && $files_processed_count -gt $FILE_LIMIT ]] && { echo "Reached file limit"; break 3; }
890

            
Bogdan Timofte authored 3 months ago
891
            if [[ $DRY_RUN -eq 1 && $files_shown -ge $max_files_to_show ]]; then
Bogdan Timofte authored 2 weeks ago
892
                [[ $files_shown -eq $max_files_to_show ]] && echo "... (limiting output in dry-run mode)"
893
                ((files_shown++))
Bogdan Timofte authored 3 months ago
894
            else
Bogdan Timofte authored 2 weeks ago
895
                [[ $DRY_RUN -eq 1 ]] && ((files_shown++))
896
                process_file "$file"
Bogdan Timofte authored 3 months ago
897
            fi
898
        done < <(find "$dir" -type f -iname "$ext" -print0 2>/dev/null)
899
    done
900
done <<< "$camera_dirs"
901

            
902
# Summary
Bogdan Timofte authored 2 weeks ago
903
log_message "Import completed: $PROCESSED_FILES/$TOTAL_FILES files processed successfully" "info"
904
[[ $ERROR_FILES -gt 0 ]] && log_message "Import had errors: $ERROR_FILES files failed" "warning"
Bogdan Timofte authored 3 months ago
905

            
906
echo ""
907
echo "=== Import Summary ==="
Bogdan Timofte authored 2 weeks ago
908
echo "Total files found: $TOTAL_FILES"
909
echo "Successfully processed: $PROCESSED_FILES"
910
echo "Skipped: $SKIPPED_FILES"
911
echo "Errors: $ERROR_FILES"
912
[[ $glv_count -gt 0 ]] && echo "GLV files cleaned up: $glv_count"
Bogdan Timofte authored 3 months ago
913

            
Bogdan Timofte authored 2 weeks ago
914
# Cleanup empty directories
915
if [[ $KEEP_EMPTY_DIRS -eq 1 && $DRY_RUN -eq 0 ]]; then
916
    log_message "Cleaning up empty directories..." "info"
917
    find "$DESTINATION" -type d -empty -not -path "$DESTINATION" -delete 2>/dev/null || true
Bogdan Timofte authored 3 months ago
918
fi
Bogdan Timofte authored 2 weeks ago
919

            
920
[[ $ERROR_FILES -gt 0 ]] && exit 1 || exit 0