MediaImporter
/
media-importer.sh
1 contributor
#!/bin/bash
# Standalone Media Importer
# Version: 1.0
# A comprehensive media file organizer that sorts photos and videos by date
# with various organization patterns and timezone handling
VERSION="1.0"
SCRIPT_NAME="Standalone Media Importer"
# Default values
# Default organization: 'ymd' (single folder per day yyyy-mm-dd). Override with -o/--organization.
ORGANIZATION="ymd"
FILENAME_MODE="full" # options: auto, full, orig
COLLECT_UNSORTABLE=0
SOURCE_PATTERNS=()
DESTINATION=""
KEEP_ORIGINALS=0
VERIFY_MODE="size" # options: size, strict, none
DATE_SOURCE="auto" # options: auto, exif, filesystem
SYNC_METADATA=0 # when 1, write reconstructed date into destination metadata
UNATTENDED=0 # when 1, never prompt; destination conflicts get numeric suffixes
DRY_RUN=0
VERBOSE=0
CLEANUP_EMPTY_DIRS=1
CONFLICT_APPLY_ALL="" # suffix|skip after an interactive "all similar" choice
RESOLVED_DESTINATION_PATH=""
RESERVED_DESTINATION_PATHS=()
# Counters and statistics
TOTAL_FILES=0
PROCESSED_FILES=0
SKIPPED_FILES=0
ERROR_FILES=0
TOTAL_SIZE=0
PROCESSED_SIZE=0
START_TIME=$(date +%s)
CURRENT_FILE_INDEX=0
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
# BLUE is used for informational/verbose messages. Dark terminal themes may render the
# default blue hard to read, so use a brighter cyan by default and allow users to
# override via the VERBOSE_COLOR environment variable (must be a terminal escape seq).
if [[ -n "${VERBOSE_COLOR:-}" ]]; then
BLUE="$VERBOSE_COLOR"
else
BLUE=$'\033[1;36m' # bright cyan
fi
NC='\033[0m' # No Color
# Function to print colored output
print_color() {
local color="$1"
local message="$2"
echo -e "${color}${message}${NC}"
}
# Function to log messages with timestamp
log_message() {
local message="$1"
local level="${2:-INFO}"
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
case "$level" in
"ERROR")
print_color "$RED" "[$timestamp] ERROR: $message" >&2
;;
"WARNING")
print_color "$YELLOW" "[$timestamp] WARNING: $message"
;;
"SUCCESS")
print_color "$GREEN" "[$timestamp] SUCCESS: $message"
;;
"INFO")
if [[ $VERBOSE -eq 1 ]]; then
print_color "$BLUE" "[$timestamp] INFO: $message"
fi
;;
*)
echo "[$timestamp] $message"
;;
esac
}
# Function to display help
show_help() {
cat << EOF
$SCRIPT_NAME v$VERSION
Usage:
$0 [OPTIONS]
What it does:
Sorts photos and videos into dated folders (by year/month/day/hour) and
generates filenames from the file's creation timestamp or preserves the
original name.
Options:
-o, --organization PATTERN y|m|d|h|ym|ymd (default: ymd)
-F, --filename-mode MODE auto|full|orig (default: full)
-s, --source PATH File or directory to process (repeatable). Default: cwd
-d, --destination PATH Destination folder. Required when multiple -s are given.
-k, --keep-originals Copy files instead of moving
--verify-mode MODE size|strict|none (default: size)
--date-source SOURCE auto|exif|filesystem (default: auto)
--sync-metadata Write chosen date into destination metadata (automatic for GoPro filesystem dates)
--unattended Never prompt; resolve destination conflicts with numeric suffixes
--collect-unsortable Put files without dates into DEST/unsortable
--keep-empty-dirs Keep empty directories after processing
--dry-run Show actions without changing files
-v, --verbose Verbose output
-h, --help Show this help
--version Show version
Examples:
$0 -s /path/to/photos
$0 -s /path/to/DCIM -d /mnt/sorted --dry-run
Dependencies:
exiftool (required). mediainfo and file are optional.
EOF
}
# Central media extensions list (used by find functions)
MEDIA_EXTENSIONS=("*.jpg" "*.jpeg" "*.png" "*.tiff" "*.tif" "*.cr2" "*.nef" "*.arw" "*.dng" "*.raw" "*.mp4" "*.mov" "*.avi" "*.mts" "*.m2ts" "*.mkv" "*.wmv" "*.3gp" "*.m4v")
# Function to show version
show_version() {
echo "$SCRIPT_NAME v$VERSION"
echo "A comprehensive media file organizer with timezone support"
}
# Function to check dependencies
check_dependencies() {
local missing_deps=()
# Check for required dependencies
if ! command -v exiftool &> /dev/null; then
missing_deps+=("exiftool")
fi
# Check for optional dependencies
local optional_missing=()
if ! command -v mediainfo &> /dev/null; then
optional_missing+=("mediainfo")
fi
if ! command -v file &> /dev/null; then
optional_missing+=("file")
fi
if [[ ${#missing_deps[@]} -gt 0 ]]; then
print_color "$RED" "ERROR: Missing required dependencies:"
for dep in "${missing_deps[@]}"; do
echo " - $dep"
done
echo ""
echo "Installation instructions:"
echo " macOS: brew install exiftool"
echo " Ubuntu/Debian: sudo apt-get install libimage-exiftool-perl"
echo " CentOS/RHEL: sudo yum install perl-Image-ExifTool"
echo " Arch: sudo pacman -S perl-image-exiftool"
exit 1
fi
if [[ ${#optional_missing[@]} -gt 0 && $VERBOSE -eq 1 ]]; then
log_message "Optional dependencies not found (functionality may be limited): ${optional_missing[*]}" "WARNING"
fi
log_message "All required dependencies found" "SUCCESS"
}
# Determine filesystem/device ID for a path (portable between Linux and macOS)
get_dev() {
local path="$1"
if [[ -z "$path" ]]; then
path="."
fi
# Prefer GNU stat if available
if stat --version >/dev/null 2>&1; then
stat -c %d "$path" 2>/dev/null || stat -c %i "$path" 2>/dev/null || echo ""
else
# BSD/macOS stat
stat -f %d "$path" 2>/dev/null || stat -f %i "$path" 2>/dev/null || echo ""
fi
}
# Function to get file size in bytes (portable between Linux and macOS)
get_file_size() {
local file="$1"
if [[ -f "$file" ]]; then
# Try GNU stat
if stat -c%s "$file" >/dev/null 2>&1; then
stat -c%s "$file" 2>/dev/null || stat -f%z "$file" 2>/dev/null
elif stat -f%z "$file" >/dev/null 2>&1; then
stat -f%z "$file" 2>/dev/null
else
# Fallback to ls
ls -ln "$file" | awk '{print $5}'
fi
else
echo "0"
fi
}
# (Removed checksum/prefix/conflict helper functions to revert to pre-conflict-resolution behavior)
# Safe move/copy helpers: filter out benign "set flags (was: ...): Operation not supported"
# which appears when moving files onto filesystems that don't support BSD file flags
# (macOS mv may try to preserve flags and print this warning while still succeeding).
safe_mv() {
local src="$1" dst="$2"
# Redirect stderr through a filter that removes the known benign message
mv "$src" "$dst" 2> >(grep -v "set flags (was:" >&2)
return $?
}
destination_path_reserved() {
local candidate="$1"
local reserved_path
for reserved_path in "${RESERVED_DESTINATION_PATHS[@]}"; do
if [[ "$reserved_path" == "$candidate" ]]; then
return 0
fi
done
return 1
}
destination_path_unavailable() {
local candidate="$1"
[[ -e "$candidate" ]] || destination_path_reserved "$candidate"
}
reserve_destination_path() {
local candidate="$1"
if [[ -n "$candidate" ]] && ! destination_path_reserved "$candidate"; then
RESERVED_DESTINATION_PATHS+=("$candidate")
fi
}
prompt_destination_conflict_choice() {
local source_file="$1"
local desired_path="$2"
local choice
if [[ ! -r /dev/tty || ! -w /dev/tty ]]; then
return 1
fi
{
print_color "$YELLOW" "Destination already exists:"
echo " Source: $source_file"
echo " Destination: $desired_path"
echo ""
echo "Choose conflict action:"
echo " [s] suffix once"
echo " [S] suffix for all similar conflicts"
echo " [k] skip once"
echo " [K] skip all similar conflicts"
echo " [a] abort import"
} > /dev/tty
while true; do
printf "Action [s/S/k/K/a]: " > /dev/tty
IFS= read -r choice < /dev/tty || return 1
case "$choice" in
s|"")
echo "suffix"
return 0
;;
S)
echo "suffix_all"
return 0
;;
k)
echo "skip"
return 0
;;
K)
echo "skip_all"
return 0
;;
a|A)
echo "abort"
return 0
;;
*)
print_color "$YELLOW" "Please choose s, S, k, K, or a." > /dev/tty
;;
esac
done
}
resolve_destination_conflict() {
local desired_path="$1"
local source_file="$2"
local resolved_path choice
RESOLVED_DESTINATION_PATH=""
if [[ -z "$desired_path" ]]; then
return 1
fi
if ! destination_path_unavailable "$desired_path"; then
RESOLVED_DESTINATION_PATH="$desired_path"
reserve_destination_path "$RESOLVED_DESTINATION_PATH"
return 0
fi
if [[ "$CONFLICT_APPLY_ALL" == "skip" ]]; then
return 3
fi
if [[ "$CONFLICT_APPLY_ALL" == "suffix" || $UNATTENDED -eq 1 ]]; then
resolved_path=$(ensure_unique_destination_path "$desired_path")
if [[ -z "$resolved_path" ]]; then
return 1
fi
RESOLVED_DESTINATION_PATH="$resolved_path"
reserve_destination_path "$RESOLVED_DESTINATION_PATH"
return 0
fi
choice=$(prompt_destination_conflict_choice "$source_file" "$desired_path")
if [[ $? -ne 0 || -z "$choice" ]]; then
log_message "Cannot prompt for destination conflict; using unattended numeric suffix mode" "WARNING"
resolved_path=$(ensure_unique_destination_path "$desired_path")
if [[ -z "$resolved_path" ]]; then
return 1
fi
RESOLVED_DESTINATION_PATH="$resolved_path"
reserve_destination_path "$RESOLVED_DESTINATION_PATH"
return 0
fi
case "$choice" in
suffix)
resolved_path=$(ensure_unique_destination_path "$desired_path")
;;
suffix_all)
CONFLICT_APPLY_ALL="suffix"
resolved_path=$(ensure_unique_destination_path "$desired_path")
;;
skip)
return 3
;;
skip_all)
CONFLICT_APPLY_ALL="skip"
return 3
;;
abort)
return 4
;;
*)
return 1
;;
esac
if [[ -z "$resolved_path" ]]; then
return 1
fi
RESOLVED_DESTINATION_PATH="$resolved_path"
reserve_destination_path "$RESOLVED_DESTINATION_PATH"
return 0
}
extract_filesystem_date() {
# Returns yyyy-mm-dd hh:mm:ss based on filesystem mtime.
# We intentionally use mtime (not birthtime) because birthtime isn't preserved by copies
# across filesystems, while mtime can be preserved via `cp -p`.
local file="$1"
if [[ ! -e "$file" ]]; then
return 2
fi
local epoch=""
if [[ "$OSTYPE" == "darwin"* ]]; then
epoch=$(stat -f %m "$file" 2>/dev/null || echo "")
[[ -n "$epoch" ]] || return 2
date -j -r "$epoch" "+%Y-%m-%d %H:%M:%S" 2>/dev/null || return 2
return 0
else
epoch=$(stat -c %Y "$file" 2>/dev/null || echo "")
[[ -n "$epoch" ]] || return 2
date -d "@$epoch" "+%Y-%m-%d %H:%M:%S" 2>/dev/null || return 2
return 0
fi
}
filesystem_date_reference() {
local file="$1"
local dir base stem ext sidecar_ext sidecar
dir=$(dirname "$file")
base=$(basename "$file")
stem="${base%.*}"
ext="${base##*.}"
if [[ "$ext" =~ ^([Mm][Pp]4)$ ]]; then
for sidecar_ext in THM thm LRV lrv; do
sidecar="$dir/$stem.$sidecar_ext"
if [[ -f "$sidecar" ]]; then
echo "$sidecar"
return 0
fi
done
fi
echo "$file"
}
is_gopro_media_file() {
local filename
filename=$(basename "$1")
[[ "$filename" =~ ^G[HPX][0-9]{6}\.[Mm][Pp]4$ ]]
}
should_prefer_gopro_filesystem_date() {
local file="$1"
is_gopro_media_file "$file"
}
filesystem_date_source_label() {
local file="$1"
local reference="$2"
if is_gopro_media_file "$file"; then
echo "Filesystem:$(basename "$reference")"
elif [[ "$reference" != "$file" ]]; then
echo "Filesystem:$(basename "$reference")"
else
echo "Filesystem"
fi
}
date_to_exiftool_format() {
# yyyy-mm-dd hh:mm:ss -> yyyy:mm:dd hh:mm:ss
local s="$1"
if [[ "$s" =~ ^([0-9]{4})-([0-9]{2})-([0-9]{2})[[:space:]]+([0-9]{2}):([0-9]{2}):([0-9]{2})$ ]]; then
echo "${BASH_REMATCH[1]}:${BASH_REMATCH[2]}:${BASH_REMATCH[3]} ${BASH_REMATCH[4]}:${BASH_REMATCH[5]}:${BASH_REMATCH[6]}"
return 0
fi
return 1
}
sync_destination_metadata_to_date() {
local file="$1"
local date_str="$2" # yyyy-mm-dd hh:mm:ss
local exif_dt
exif_dt=$(date_to_exiftool_format "$date_str") || return 1
exiftool -overwrite_original \
"-CreateDate=$exif_dt" \
"-DateTimeOriginal=$exif_dt" \
"-DateTime=$exif_dt" \
"-ModifyDate=$exif_dt" \
"-MediaCreateDate=$exif_dt" \
"-TrackCreateDate=$exif_dt" \
"-QuickTime:CreateDate=$exif_dt" \
"-QuickTime:ModifyDate=$exif_dt" \
"$file" >/dev/null 2>&1
return 0
}
verify_synced_metadata_date() {
local file="$1"
local expected_date="$2"
local metadata_date
metadata_date=$(exiftool -api QuickTimeUTC=0 -s -s -s -QuickTime:CreateDate "$file" 2>/dev/null | head -1)
if [[ -z "$metadata_date" ]]; then
metadata_date=$(exiftool -api QuickTimeUTC=0 -s -s -s -CreateDate "$file" 2>/dev/null | head -1)
fi
if [[ "$metadata_date" =~ ^([0-9]{4}):([0-9]{2}):([0-9]{2})[[:space:]]+([0-9]{2}):([0-9]{2}):([0-9]{2}) ]]; then
metadata_date="${BASH_REMATCH[1]}-${BASH_REMATCH[2]}-${BASH_REMATCH[3]} ${BASH_REMATCH[4]}:${BASH_REMATCH[5]}:${BASH_REMATCH[6]}"
fi
if [[ "$metadata_date" != "$expected_date" ]]; then
log_message "Destination metadata sync mismatch: expected $expected_date, got ${metadata_date:-none} for $file" "ERROR"
return 1
fi
return 0
}
should_sync_imported_metadata() {
local original_filename="$1"
local date_source="$2"
if [[ $SYNC_METADATA -eq 1 ]]; then
return 0
fi
if [[ "$date_source" == Filesystem* ]] && is_gopro_media_file "$original_filename"; then
return 0
fi
return 1
}
verify_copied_file() {
local src="$1"
local dst="$2"
local expected_date="$3"
if [[ ! -f "$dst" ]]; then
log_message "Verified copy missing at destination: $dst" "ERROR"
return 1
fi
local src_size dst_size
src_size=$(get_file_size "$src")
dst_size=$(get_file_size "$dst")
if [[ "$src_size" != "$dst_size" ]]; then
log_message "Size mismatch after copy: $src ($src_size) != $dst ($dst_size)" "ERROR"
return 1
fi
if [[ "$VERIFY_MODE" == "strict" ]]; then
if ! cmp -s "$src" "$dst"; then
log_message "Content mismatch after copy: $src -> $dst" "ERROR"
return 1
fi
elif [[ "$VERIFY_MODE" == "none" ]]; then
return 0
fi
if [[ -n "$expected_date" ]]; then
local destination_date_info
destination_date_info=$(extract_file_date "$dst")
local extract_status=$?
if [[ $extract_status -ne 0 || -z "$destination_date_info" ]]; then
log_message "Destination metadata validation failed: $dst" "ERROR"
return 1
fi
local destination_date="${destination_date_info%|*}"
if [[ "$destination_date" != "$expected_date" ]]; then
log_message "Destination metadata mismatch: expected $expected_date, got $destination_date for $dst" "ERROR"
return 1
fi
fi
return 0
}
remove_source_file() {
local src="$1"
rm -f "$src"
}
copy_with_verification() {
local src="$1"
local dst="$2"
local expected_date="$3"
if [[ -e "$dst" ]]; then
log_message "Refusing to overwrite existing destination: $dst" "ERROR"
return 1
fi
local dst_dir tmp
dst_dir=$(dirname "$dst")
tmp=$(mktemp "$dst_dir/.media-importer.$(basename "$dst").tmp.XXXXXX") || return 1
rm -f "$tmp"
if ! safe_cp "$src" "$tmp"; then
rm -f "$tmp"
return 1
fi
if ! verify_copied_file "$src" "$tmp" "$expected_date"; then
rm -f "$tmp"
return 1
fi
if [[ -e "$dst" ]]; then
log_message "Destination appeared during copy, refusing to overwrite: $dst" "ERROR"
rm -f "$tmp"
return 1
fi
if ! safe_mv "$tmp" "$dst"; then
rm -f "$tmp"
return 1
fi
if [[ ! -f "$dst" ]]; then
log_message "Copied file missing after final move: $dst" "ERROR"
return 1
fi
return 0
}
verified_move_file() {
local src="$1"
local dst="$2"
local expected_date="$3"
if ! copy_with_verification "$src" "$dst" "$expected_date"; then
return 1
fi
if ! remove_source_file "$src"; then
log_message "Copied and verified destination, but failed to remove source: $src" "ERROR"
return 1
fi
return 0
}
# Function to format file size
format_size() {
local size=$1
if (( size < 1024 )); then
echo "${size}B"
elif (( size < 1048576 )); then
echo "$(( size / 1024 ))KB"
elif (( size < 1073741824 )); then
echo "$(( size / 1048576 ))MB"
else
echo "$(( size / 1073741824 ))GB"
fi
}
format_duration() {
local total_seconds=$1
local hours=$((total_seconds / 3600))
local minutes=$(((total_seconds % 3600) / 60))
local seconds=$((total_seconds % 60))
if (( hours > 0 )); then
printf "%dh %02dm %02ds" "$hours" "$minutes" "$seconds"
elif (( minutes > 0 )); then
printf "%dm %02ds" "$minutes" "$seconds"
else
printf "%ds" "$seconds"
fi
}
format_processing_rate() {
local files_count="$1"
local bytes_count="$2"
local elapsed_seconds="$3"
awk -v files="$files_count" -v bytes="$bytes_count" -v seconds="$elapsed_seconds" '
BEGIN {
if (seconds <= 0 || files <= 0) {
exit
}
files_per_second = files / seconds
files_per_minute = files * 60 / seconds
mb_per_second = bytes / seconds / 1048576
if (files_per_second >= 1) {
printf "%.2f files/sec, %.2f MB/sec", files_per_second, mb_per_second
} else {
printf "%.2f files/min, %.2f MB/sec", files_per_minute, mb_per_second
}
}
'
}
# Function to extract date from file
extract_file_date() {
local file="$1"
local create_date=""
local date_source=""
local exif_found=0
# Filesystem authoritative mode, and GoPro media in auto mode.
# GoPro fallback order is THM, LRV, then the MP4 filesystem timestamp.
if [[ "$DATE_SOURCE" == "filesystem" ]] || { [[ "$DATE_SOURCE" == "auto" ]] && should_prefer_gopro_filesystem_date "$file"; }; then
local filesystem_reference
filesystem_reference=$(filesystem_date_reference "$file")
create_date=$(extract_filesystem_date "$filesystem_reference") || return 2
echo "$create_date|$(filesystem_date_source_label "$file" "$filesystem_reference")"
return 0
fi
# Try to get creation date from EXIF data
local exif_output=$(exiftool -G1 -s -CreateDate -DateTimeOriginal -DateTime "$file" 2>/dev/null)
if [[ -n "$exif_output" ]]; then
# Parse the exiftool output to find the best date
while IFS= read -r line; do
if [[ "$line" =~ ^\[([^\]]+)\][[:space:]]*([^:]+)[[:space:]]*:[[:space:]]*(.+)$ ]]; then
local group="${BASH_REMATCH[1]}"
local tag="${BASH_REMATCH[2]}"
local value="${BASH_REMATCH[3]}"
# Trim spaces from tag name
tag=$(echo "$tag" | sed 's/[[:space:]]*$//')
# Prefer DateTimeOriginal, then CreateDate, then DateTime
if [[ "$tag" == "DateTimeOriginal" && -z "$create_date" ]]; then
create_date="$value"
date_source="$group:$tag"
exif_found=1
elif [[ "$tag" == "CreateDate" && "$date_source" != *"DateTimeOriginal"* ]]; then
create_date="$value"
date_source="$group:$tag"
exif_found=1
elif [[ "$tag" == "DateTime" && -z "$create_date" ]]; then
create_date="$value"
date_source="$group:$tag"
exif_found=1
fi
fi
done <<< "$exif_output"
fi
# If no EXIF date found, try mediainfo for video files
if [[ -z "$create_date" ]] && command -v mediainfo &> /dev/null; then
local media_date=$(mediainfo --Inform="General;%Recorded_Date%" "$file" 2>/dev/null)
if [[ -n "$media_date" && "$media_date" != "0000-00-00 00:00:00" ]]; then
create_date="$media_date"
date_source="MediaInfo:Recorded_Date"
fi
fi
# In auto mode, if metadata is missing/unreliable, fall back to filesystem timestamps
if [[ -z "$create_date" && "$DATE_SOURCE" == "auto" ]]; then
local filesystem_reference
filesystem_reference=$(filesystem_date_reference "$file")
create_date=$(extract_filesystem_date "$filesystem_reference") || return 2
date_source=$(filesystem_date_source_label "$file" "$filesystem_reference")
fi
# If no EXIF or mediainfo date found, return failure
if [[ -z "$create_date" ]]; then
return 2 # No date metadata found
fi
# Convert EXIF format (YYYY:MM:DD HH:MM:SS) to standard format
# Always output as yyyy-mm-dd hh:mm:ss (pad single digits)
if [[ "$create_date" =~ ^([0-9]{4}):([0-9]{1,2}):([0-9]{1,2})[[:space:]]*([0-9]{1,2}):([0-9]{1,2}):([0-9]{1,2})$ ]]; then
year="${BASH_REMATCH[1]}"
month=$(printf "%02d" "$((10#${BASH_REMATCH[2]}))")
day=$(printf "%02d" "$((10#${BASH_REMATCH[3]}))")
hour=$(printf "%02d" "$((10#${BASH_REMATCH[4]}))")
minute=$(printf "%02d" "$((10#${BASH_REMATCH[5]}))")
second=$(printf "%02d" "$((10#${BASH_REMATCH[6]}))")
create_date="$year-$month-$day $hour:$minute:$second"
else
# Try to convert yyyy-mm-dd hh:mm:ss (already correct)
if [[ "$create_date" =~ ^([0-9]{4})-([0-9]{2})-([0-9]{2})[[:space:]]*([0-9]{2}):([0-9]{2}):([0-9]{2})$ ]]; then
# Already correct
:
else
print_color "$RED" "Error: Cannot parse date '$create_date'" >&2
return 2
fi
fi
# For QuickTime files, the CreateDate is in UTC and needs conversion to local time
if [[ "$date_source" == *"QuickTime"* ]]; then
# Convert UTC time to local time
if [[ "$OSTYPE" == "darwin"* ]]; then
# On macOS, use TZ=UTC to interpret the input time as UTC
local utc_timestamp=$(TZ=UTC date -j -f "%Y-%m-%d %H:%M:%S" "$create_date" "+%s" 2>/dev/null)
if [[ -n "$utc_timestamp" ]]; then
create_date=$(date -j -r "$utc_timestamp" "+%Y-%m-%d %H:%M:%S" 2>/dev/null)
date_source="$date_source (converted from UTC)"
fi
else
local utc_timestamp=$(date -d "$create_date UTC" "+%s" 2>/dev/null)
if [[ -n "$utc_timestamp" ]]; then
create_date=$(date -d "@$utc_timestamp" "+%Y-%m-%d %H:%M:%S" 2>/dev/null)
date_source="$date_source (converted from UTC)"
fi
fi
fi
echo "$create_date|$date_source"
return 0
}
# Function to generate destination path based on organization pattern
generate_destination_path() {
local date_str="$1"
local original_filename="$2"
local base_destination="$3"
# Extract date components - handle both GNU and BSD date
local year month day hour minute second
if [[ "$OSTYPE" == "darwin"* ]]; then
# macOS (BSD date) - use more robust regex (allow single-digit month/day, tolerate extra spaces)
if [[ "$date_str" =~ ^([0-9]{4})-([0-9]{1,2})-([0-9]{1,2})[[:space:]]*([0-9]{1,2}):([0-9]{1,2}):([0-9]{1,2})$ ]]; then
year="${BASH_REMATCH[1]}"
month=$(printf "%02d" "$((10#${BASH_REMATCH[2]}))")
day=$(printf "%02d" "$((10#${BASH_REMATCH[3]}))")
hour=$(printf "%02d" "$((10#${BASH_REMATCH[4]}))")
minute=$(printf "%02d" "$((10#${BASH_REMATCH[5]}))")
second=$(printf "%02d" "$((10#${BASH_REMATCH[6]}))")
else
return 1
fi
else
# Linux (GNU date)
year=$(date -d "$date_str" "+%Y" 2>/dev/null)
month=$(date -d "$date_str" "+%m" 2>/dev/null)
day=$(date -d "$date_str" "+%d" 2>/dev/null)
hour=$(date -d "$date_str" "+%H" 2>/dev/null)
minute=$(date -d "$date_str" "+%M" 2>/dev/null)
second=$(date -d "$date_str" "+%S" 2>/dev/null)
fi
if [[ -z "$year" || -z "$month" || -z "$day" ]]; then
return 1
fi
# Get file extension
local extension="${original_filename##*.}"
local lowercase_ext=$(echo "$extension" | tr '[:upper:]' '[:lower:]')
# Generate path and filename based on organization pattern
local dir_path=""
local filename=""
# If no organization specified, use flat destination (base) and choose filename per mode
if [[ -z "$ORGANIZATION" ]]; then
dir_path="$base_destination"
if [[ "$FILENAME_MODE" == "orig" ]]; then
filename="$original_filename"
else
# full or auto both map to full date for flat layout
filename="${year}-${month}-${day}_${hour}-${minute}-${second}.${lowercase_ext}"
fi
echo "$dir_path/$filename"
return 0
fi
case "$ORGANIZATION" in
"y")
dir_path="$base_destination/$year"
filename="${month}-${day}_${hour}-${minute}-${second}.${lowercase_ext}"
;;
"m")
dir_path="$base_destination/$year/$month"
filename="${day}_${hour}-${minute}-${second}.${lowercase_ext}"
;;
"d")
dir_path="$base_destination/$year/$month/$day"
filename="${month}-${day}_${hour}-${minute}-${second}.${lowercase_ext}"
;;
"h")
dir_path="$base_destination/$year/$month/$day/$hour"
filename="${minute}-${second}.${lowercase_ext}"
;;
"ym")
# Single folder per month named yyyy-mm; filename includes day and time
dir_path="$base_destination/${year}-${month}"
filename="${day}_${hour}-${minute}-${second}.${lowercase_ext}"
;;
"ymd")
# Single folder per day named yyyy-mm-dd; filename is time
dir_path="$base_destination/${year}-${month}-${day}"
filename="${hour}-${minute}-${second}.${lowercase_ext}"
;;
*)
log_message "Invalid organization pattern: $ORGANIZATION" "ERROR"
return 1
;;
esac
# Apply filename mode overrides
case "$FILENAME_MODE" in
orig)
filename="$original_filename"
;;
full)
filename="${year}-${month}-${day}_${hour}-${minute}-${second}.${lowercase_ext}"
;;
auto)
# keep the auto-generated filename from the organization case
;;
*)
# fallback to full
filename="${year}-${month}-${day}_${hour}-${minute}-${second}.${lowercase_ext}"
;;
esac
echo "$dir_path/$filename"
return 0
}
# Function to find files matching patterns
find_source_files() {
# Emit absolute file paths for all media files under sources, excluding DESTINATION when it is inside a source
local abs_dest=""
if [[ -n "$DESTINATION" ]]; then
abs_dest=$(cd "$DESTINATION" 2>/dev/null && pwd) || abs_dest="$DESTINATION"
fi
# Build -iname expression for find
local ext_expr=""
for ext in "${MEDIA_EXTENSIONS[@]}"; do
if [[ -n "$ext_expr" ]]; then
ext_expr="$ext_expr -o"
fi
ext_expr="$ext_expr -iname $ext"
done
if [[ ${#SOURCE_PATTERNS[@]} -eq 0 ]]; then
# Default: scan current directory
local start_dot="."
local abs_current
abs_current=$(pwd)
local find_cmd=(find -L "$start_dot" -type f)
# If dest is inside cwd, add exclusion
if [[ -n "$abs_dest" && "$abs_dest" == "$abs_current"* ]]; then
find_cmd+=( ! -path "$abs_dest/*" ! -path "$abs_dest" )
fi
# Add expression
# shellcheck disable=SC2068
"${find_cmd[@]}" ! -name '._*' \( $ext_expr \) 2>/dev/null || true
else
# Scan each provided source
for src in "${SOURCE_PATTERNS[@]}"; do
if [[ -f "$src" ]]; then
if [[ "$(basename "$src")" == ._* ]]; then
continue
fi
# single file - skip if it's inside dest
local abs_file
abs_file=$(cd "$(dirname "$src")" 2>/dev/null && pwd)/$(basename "$src")
if [[ -n "$abs_dest" && "$abs_file" == "$abs_dest"* ]]; then
continue
fi
echo "$abs_file"
elif [[ -d "$src" ]]; then
local abs_src
abs_src=$(cd "$src" 2>/dev/null && pwd)
if [[ -n "$abs_src" ]]; then
if [[ -n "$abs_dest" && "$abs_dest" == "$abs_src"* ]]; then
find -L "$abs_src" -type f ! -name '._*' \( $ext_expr \) ! -path "$abs_dest/*" ! -path "$abs_dest" 2>/dev/null || true
else
find -L "$abs_src" -type f ! -name '._*' \( $ext_expr \) 2>/dev/null || true
fi
else
print_color "$YELLOW" "Warning: Could not resolve source directory: $src"
fi
else
print_color "$YELLOW" "Warning: Source path does not exist or is not a file/directory: $src"
fi
done
fi
}
# Function to process a single file
process_file() {
local file="$1"
local file_size=$(get_file_size "$file")
local file_label
file_label="$(basename "$file")"
TOTAL_SIZE=$((TOTAL_SIZE + file_size))
if [[ $TOTAL_FILES -gt 0 && $CURRENT_FILE_INDEX -gt 0 ]]; then
print_color "$BLUE" "Processing [$CURRENT_FILE_INDEX/$TOTAL_FILES]: $file_label ($(format_size "$file_size"))"
else
print_color "$BLUE" "Processing: $file_label ($(format_size "$file_size"))"
fi
log_message "Processing: $file" "INFO"
# Extract date information
local date_info=$(extract_file_date "$file")
local extract_status=$?
if [[ $extract_status -eq 2 ]]; then
if [[ $COLLECT_UNSORTABLE -eq 1 ]]; then
local unsortable_dir="$DESTINATION/unsortable"
mkdir -p "$unsortable_dir"
local unsortable_path="$unsortable_dir/$(basename "$file")"
local desired_unsortable_path="$unsortable_path"
local unsortable_conflict_status
resolve_destination_conflict "$unsortable_path" "$file"
unsortable_conflict_status=$?
if [[ $unsortable_conflict_status -eq 0 ]]; then
unsortable_path="$RESOLVED_DESTINATION_PATH"
if [[ "$unsortable_path" != "$desired_unsortable_path" ]]; then
log_message "Destination already exists or is already planned: $desired_unsortable_path - using: $unsortable_path" "WARNING"
fi
elif [[ $unsortable_conflict_status -eq 3 ]]; then
log_message "Destination conflict skipped: $desired_unsortable_path" "WARNING"
SKIPPED_FILES=$((SKIPPED_FILES + 1))
return 1
elif [[ $unsortable_conflict_status -eq 4 ]]; then
log_message "Import aborted by user at destination conflict: $desired_unsortable_path" "ERROR"
ERROR_FILES=$((ERROR_FILES + 1))
FATAL_ERROR=1
return 2
else
log_message "Could not resolve a unique destination path for $file (wanted: $desired_unsortable_path)" "ERROR"
ERROR_FILES=$((ERROR_FILES + 1))
return 1
fi
if [[ $DRY_RUN -eq 1 ]]; then
print_color "$BLUE" "Would move unsortable: $file -> $unsortable_path"
else
if verified_move_file "$file" "$unsortable_path" ""; then
log_message "Unsortable: $file -> $unsortable_path" "SUCCESS"
else
log_message "Failed to move unsortable file after verification: $file" "ERROR"
fi
fi
SKIPPED_FILES=$((SKIPPED_FILES + 1))
else
log_message "Could not extract date from $file - skipping" "WARNING"
SKIPPED_FILES=$((SKIPPED_FILES + 1))
fi
return 1
elif [[ $extract_status -ne 0 ]] || [[ -z "$date_info" ]]; then
log_message "Could not extract date from $file - skipping" "WARNING"
SKIPPED_FILES=$((SKIPPED_FILES + 1))
return 1
fi
local date_str="${date_info%|*}"
local date_source="${date_info#*|}"
log_message "Date: $date_str (from $date_source)" "INFO"
# Generate destination path
local original_basename
original_basename="$(basename "$file")"
local dest_path
dest_path=$(generate_destination_path "$date_str" "$original_basename" "$DESTINATION")
if [[ $? -ne 0 ]] || [[ -z "$dest_path" ]]; then
log_message "Could not generate destination path for $file" "ERROR"
ERROR_FILES=$((ERROR_FILES + 1))
FATAL_ERROR=1
return 2
fi
local desired_dest_path="$dest_path"
local conflict_status
resolve_destination_conflict "$dest_path" "$file"
conflict_status=$?
if [[ $conflict_status -eq 0 ]]; then
dest_path="$RESOLVED_DESTINATION_PATH"
elif [[ $conflict_status -eq 3 ]]; then
log_message "Destination conflict skipped: $desired_dest_path" "WARNING"
SKIPPED_FILES=$((SKIPPED_FILES + 1))
return 1
elif [[ $conflict_status -eq 4 ]]; then
log_message "Import aborted by user at destination conflict: $desired_dest_path" "ERROR"
ERROR_FILES=$((ERROR_FILES + 1))
FATAL_ERROR=1
return 2
else
log_message "Could not resolve a unique destination path for $file (wanted: $desired_dest_path)" "ERROR"
ERROR_FILES=$((ERROR_FILES + 1))
return 1
fi
if [[ "$dest_path" != "$desired_dest_path" ]]; then
log_message "Destination already exists or is already planned: $desired_dest_path - using: $dest_path" "WARNING"
fi
local dest_dir
dest_dir=$(dirname "$dest_path")
if [[ $DRY_RUN -eq 1 ]]; then
if [[ $KEEP_ORIGINALS -eq 1 ]]; then
print_color "$BLUE" "Would copy: $file -> $dest_path"
else
print_color "$BLUE" "Would move: $file -> $dest_path"
fi
if should_sync_imported_metadata "$original_basename" "$date_source"; then
print_color "$BLUE" "Would sync metadata date: $dest_path -> $date_str"
fi
PROCESSED_FILES=$((PROCESSED_FILES + 1))
PROCESSED_SIZE=$((PROCESSED_SIZE + file_size))
return 0
fi
# Create destination directory
if ! mkdir -p "$dest_dir"; then
log_message "Could not create directory: $dest_dir" "ERROR"
ERROR_FILES=$((ERROR_FILES + 1))
return 1
fi
local sync_metadata_after_copy=0
local verification_date="$date_str"
if should_sync_imported_metadata "$original_basename" "$date_source"; then
sync_metadata_after_copy=1
verification_date=""
fi
# Copy or move file using safe helpers after destination conflicts are resolved.
if [[ $KEEP_ORIGINALS -eq 1 ]]; then
if copy_with_verification "$file" "$dest_path" "$verification_date"; then
if [[ $sync_metadata_after_copy -eq 1 ]]; then
if ! sync_destination_metadata_to_date "$dest_path" "$date_str"; then
log_message "Failed to sync destination metadata timestamps: $dest_path" "ERROR"
ERROR_FILES=$((ERROR_FILES + 1))
return 1
elif ! verify_synced_metadata_date "$dest_path" "$date_str"; then
ERROR_FILES=$((ERROR_FILES + 1))
return 1
fi
fi
log_message "Copied: $file -> $dest_path" "SUCCESS"
PROCESSED_FILES=$((PROCESSED_FILES + 1))
PROCESSED_SIZE=$((PROCESSED_SIZE + file_size))
return 0
else
log_message "Failed to copy or verify destination: $file -> $dest_path" "ERROR"
ERROR_FILES=$((ERROR_FILES + 1))
return 1
fi
else
if [[ $sync_metadata_after_copy -eq 1 ]]; then
if copy_with_verification "$file" "$dest_path" "$verification_date"; then
if ! sync_destination_metadata_to_date "$dest_path" "$date_str"; then
log_message "Failed to sync destination metadata timestamps: $dest_path" "ERROR"
ERROR_FILES=$((ERROR_FILES + 1))
return 1
fi
if ! verify_synced_metadata_date "$dest_path" "$date_str"; then
ERROR_FILES=$((ERROR_FILES + 1))
return 1
fi
if ! remove_source_file "$file"; then
log_message "Copied, verified, and synced destination, but failed to remove source: $file" "ERROR"
ERROR_FILES=$((ERROR_FILES + 1))
return 1
fi
log_message "Moved: $file -> $dest_path" "SUCCESS"
PROCESSED_FILES=$((PROCESSED_FILES + 1))
PROCESSED_SIZE=$((PROCESSED_SIZE + file_size))
return 0
else
log_message "Failed to move after copy verification: $file -> $dest_path" "ERROR"
ERROR_FILES=$((ERROR_FILES + 1))
return 1
fi
elif verified_move_file "$file" "$dest_path" "$date_str"; then
log_message "Moved: $file -> $dest_path" "SUCCESS"
if [[ $sync_metadata_after_copy -eq 1 ]]; then
if ! sync_destination_metadata_to_date "$dest_path" "$date_str"; then
log_message "Failed to sync destination metadata timestamps: $dest_path" "WARNING"
elif ! verify_synced_metadata_date "$dest_path" "$date_str"; then
log_message "Failed to verify synced destination metadata timestamps: $dest_path" "WARNING"
fi
fi
PROCESSED_FILES=$((PROCESSED_FILES + 1))
PROCESSED_SIZE=$((PROCESSED_SIZE + file_size))
return 0
else
log_message "Failed to move after copy verification: $file -> $dest_path" "ERROR"
ERROR_FILES=$((ERROR_FILES + 1))
return 1
fi
fi
}
# Function to display final report
show_report() {
local end_time=$(date +%s)
local elapsed_time=$((end_time - START_TIME))
local hours=$((elapsed_time / 3600))
local minutes=$(((elapsed_time % 3600) / 60))
local seconds=$((elapsed_time % 60))
echo ""
print_color "$GREEN" "=========================================="
print_color "$GREEN" " PROCESSING REPORT"
print_color "$GREEN" "=========================================="
echo ""
echo "Size Summary:"
printf " %-22s %s\n" "Total size found:" "$(format_size $TOTAL_SIZE)"
printf " %-22s %s\n" "Successfully processed:" "$(format_size $PROCESSED_SIZE)"
echo ""
if [[ $DRY_RUN -eq 1 ]]; then
print_color "$YELLOW" "DRY RUN MODE - No files were actually moved/copied"
elif [[ $KEEP_ORIGINALS -eq 1 ]]; then
print_color "$BLUE" "COPY MODE - Original files were preserved"
else
print_color "$GREEN" "MOVE MODE - Files were moved to destination"
fi
echo ""
print_color "$GREEN" "=========================================="
}
if [[ "${BASH_SOURCE[0]}" != "$0" ]]; then
return 0
fi
# Parse command line arguments
while [[ $# -gt 0 ]]; do
case $1 in
-o|--organization)
ORGANIZATION="$2"
# Accept new patterns: ym, ymd as well as single-letter ones
if [[ ! "$ORGANIZATION" =~ ^(y|m|d|h|ym|ymd)$ ]]; then
print_color "$RED" "Error: Invalid organization pattern. Must be one of: y, m, d, h, ym, ymd"
exit 1
fi
shift 2
;;
-F|--filename-mode)
FILENAME_MODE="$2"
if [[ ! "$FILENAME_MODE" =~ ^(auto|full|orig)$ ]]; then
print_color "$RED" "Error: Invalid filename mode. Must be one of: auto, full, orig"
exit 1
fi
shift 2
;;
# (conflict resolution option removed; let exiftool/filesystem handle naming conflicts)
--collect-unsortable)
COLLECT_UNSORTABLE=1
shift
;;
--keep-empty-dirs)
CLEANUP_EMPTY_DIRS=0
shift
;;
-s|--source)
SOURCE_PATTERNS+=("$2")
shift 2
;;
-d|--destination)
DESTINATION="$2"
shift 2
;;
-k|--keep-originals)
KEEP_ORIGINALS=1
shift
;;
--verify-mode)
VERIFY_MODE="$2"
if [[ ! "$VERIFY_MODE" =~ ^(size|strict|none)$ ]]; then
print_color "$RED" "Error: Invalid verify mode. Must be one of: size, strict, none"
exit 1
fi
shift 2
;;
--date-source)
DATE_SOURCE="$2"
if [[ ! "$DATE_SOURCE" =~ ^(auto|exif|filesystem)$ ]]; then
print_color "$RED" "Error: Invalid date source. Must be one of: auto, exif, filesystem"
exit 1
fi
shift 2
;;
--sync-metadata)
SYNC_METADATA=1
shift
;;
--unattended)
UNATTENDED=1
shift
;;
--dry-run)
DRY_RUN=1
shift
;;
-v|--verbose)
VERBOSE=1
shift
;;
-h|--help)
show_help
exit 0
;;
--version)
show_version
exit 0
;;
*)
print_color "$RED" "Error: Unknown option: $1"
echo "Use -h or --help for usage information."
exit 1
;;
esac
done
# Non-interactive execution cannot safely ask conflict questions.
if [[ $UNATTENDED -eq 0 && ( ! -t 0 || ! -t 1 ) ]]; then
UNATTENDED=1
fi
# If no organization is provided, leave ORGANIZATION empty and filename mode will decide naming
# If no source specified, default to current directory. Refuse to run when cwd is unsafe.
if [[ ${#SOURCE_PATTERNS[@]} -eq 0 ]]; then
cwd=$(pwd)
# Resolve home and root paths
home_dir="$HOME"
case "$cwd" in
"$home_dir"|"/"|"/etc"|"/var"|"/bin"|"/sbin"|"/dev"|"/proc"|"/sys"|"/run"|"/usr"|"/lib"|"/lib64"|"/boot"|"/snap")
print_color "$RED" "Refusing to run with default source in unsafe directory: $cwd"
print_color "$RED" "Please specify $cwd as source explicitly with -s or run from a safer directory."
exit 1
;;
*)
SOURCE_PATTERNS+=("$cwd")
;;
esac
fi
# Set default destination: if user didn't provide -d and a source was given, use first source + /sorted
if [[ -z "$DESTINATION" ]]; then
if [[ ${#SOURCE_PATTERNS[@]} -gt 1 ]]; then
print_color "$RED" "Error: Multiple sources specified - destination (-d|--destination) is required when using multiple sources."
echo "Use -h for help."
exit 1
fi
if [[ ${#SOURCE_PATTERNS[@]} -gt 0 ]]; then
first_source="${SOURCE_PATTERNS[0]}"
if [[ -d "$first_source" ]]; then
DESTINATION="$first_source/sorted"
elif [[ -f "$first_source" ]]; then
DESTINATION="$(dirname "$first_source")/sorted"
else
print_color "$YELLOW" "Warning: first source does not exist; defaulting destination to ./sorted"
DESTINATION="./sorted"
fi
else
DESTINATION="./sorted"
fi
fi
# Convert destination to absolute path
DESTINATION=$(cd "$(dirname "$DESTINATION")" 2>/dev/null && pwd)/$(basename "$DESTINATION") || DESTINATION=$(realpath "$DESTINATION" 2>/dev/null) || DESTINATION="$DESTINATION"
# Display configuration
print_color "$GREEN" "$SCRIPT_NAME v$VERSION"
echo ""
echo "Configuration:"
echo " Organization pattern: $ORGANIZATION"
echo " Destination: $DESTINATION"
echo " Keep originals: $([ $KEEP_ORIGINALS -eq 1 ] && echo "Yes" || echo "No")"
echo " Verify mode: $VERIFY_MODE"
echo " Date source: $DATE_SOURCE"
echo " Sync metadata: $([ $SYNC_METADATA -eq 1 ] && echo "Yes" || echo "No")"
echo " Unattended: $([ $UNATTENDED -eq 1 ] && echo "Yes" || echo "No")"
echo " Dry run: $([ $DRY_RUN -eq 1 ] && echo "Yes" || echo "No")"
echo " Keep empty dirs: $([ $CLEANUP_EMPTY_DIRS -eq 0 ] && echo "Yes" || echo "No")"
echo " Verbose: $([ $VERBOSE -eq 1 ] && echo "Yes" || echo "No")"
if [[ ${#SOURCE_PATTERNS[@]} -gt 0 ]]; then
echo " Source patterns:"
for pattern in "${SOURCE_PATTERNS[@]}"; do
echo " - $pattern"
done
else
echo " Source patterns: All media files in current directory"
fi
echo ""
# Check dependencies
check_dependencies
# Create destination directory if it doesn't exist (unless dry run)
if [[ $DRY_RUN -eq 0 ]]; then
if ! mkdir -p "$DESTINATION"; then
print_color "$RED" "Error: Cannot create destination directory: $DESTINATION"
exit 1
fi
fi
# Find all source files
print_color "$BLUE" "Scanning for media files..."
files=()
while IFS= read -r file; do
files+=("$file")
done < <(find_source_files)
if [[ ${#files[@]} -gt 0 ]]; then
IFS=$'\n' files=($(printf '%s\n' "${files[@]}" | sort))
unset IFS
fi
TOTAL_FILES=${#files[@]}
if [[ $TOTAL_FILES -eq 0 ]]; then
print_color "$YELLOW" "No media files found matching the specified patterns."
exit 0
fi
print_color "$BLUE" "Found $TOTAL_FILES media files to process"
echo ""
# Process each file
FATAL_ERROR=0
for file in "${files[@]}"; do
if [[ -f "$file" ]]; then
CURRENT_FILE_INDEX=$((CURRENT_FILE_INDEX + 1))
process_file "$file"
if [[ $FATAL_ERROR -eq 1 ]]; then
print_color "$RED" "Fatal error encountered. Stopping further processing."
break
fi
fi
done
# Clean up empty directories if requested (default behavior)
if [[ $CLEANUP_EMPTY_DIRS -eq 1 && $DRY_RUN -eq 0 ]]; then
print_color "$BLUE" "Cleaning up empty directories..."
# Find and remove empty directories under destination, but don't remove the destination itself
find "$DESTINATION" -type d -empty -not -path "$DESTINATION" -delete 2>/dev/null || true
print_color "$GREEN" "Empty directory cleanup completed"
fi
# Show final report
show_report
# Exit with appropriate code
if [[ $ERROR_FILES -gt 0 ]]; then
exit 1
else
exit 0
fi