bogdan / MediaImporter
Showing 2 changed files with 279 additions and 1 deletions
+186 -0
extract_gpmf_gps.py
View 
@@ -0,0 +1,186 @@
1 
+#!/usr/bin/env python3
2 
+"""
3 
+Extract first valid GPS fix from a GoPro GPMF binary stream.
4 
+
5 
+Two usage modes:
6 
+  python3 extract_gpmf_gps.py <gpmf.bin>   -- parse pre-extracted binary
7 
+  python3 extract_gpmf_gps.py <file.mp4>   -- extract GPMF stream via ffmpeg, then parse
8 
+
9 
+Output (stdout):  lat lon alt
10 
+  lat/lon in decimal degrees (negative = S/W)
11 
+  alt in metres above sea level
12 
+
13 
+Exit code: 0 if a valid fix was found, 1 otherwise.
14 
+"""
15 
+
16 
+import struct
17 
+import sys
18 
+import os
19 
+import subprocess
20 
+import tempfile
21 
+
22 
+
23 
+def _iter_klv(data, offset=0):
24 
+    """Yield (key, type_char, size, repeat, payload) for each KLV record."""
25 
+    end = len(data)
26 
+    while offset + 8 <= end:
27 
+        key = data[offset:offset + 4]
28 
+        if all(b == 0 for b in key):
29 
+            break
30 
+        type_char = chr(data[offset + 4])
31 
+        size = data[offset + 5]
32 
+        repeat = struct.unpack(">H", data[offset + 6:offset + 8])[0]
33 
+        total = size * repeat
34 
+        payload = data[offset + 8:offset + 8 + total]
35 
+        yield key, type_char, size, repeat, payload
36 
+        padded = (total + 3) & ~3
37 
+        offset += 8 + padded
38 
+
39 
+
40 
+def _parse_scal(type_char, payload):
41 
+    """Return list of scale factors from a SCAL record."""
42 
+    if type_char in ("s",):
43 
+        return [struct.unpack(">h", payload[i:i + 2])[0] for i in range(0, len(payload), 2)]
44 
+    if type_char in ("S",):
45 
+        return [struct.unpack(">H", payload[i:i + 2])[0] for i in range(0, len(payload), 2)]
46 
+    if type_char in ("l",):
47 
+        return [struct.unpack(">i", payload[i:i + 4])[0] for i in range(0, len(payload), 4)]
48 
+    if type_char in ("L",):
49 
+        return [struct.unpack(">I", payload[i:i + 4])[0] for i in range(0, len(payload), 4)]
50 
+    return [1]
51 
+
52 
+
53 
+def _first_gps5_fix(strm_payload):
54 
+    """
55 
+    Scan a STRM payload for a valid GPS5 fix.
56 
+    Returns (lat, lon, alt) or None.
57 
+    Valid = GPSF >= 2 (2D or 3D fix), GPSP <= 500 (DOP*100), non-zero coords.
58 
+    """
59 
+    scal = None
60 
+    gpsf = 0
61 
+    gpsp = 9999
62 
+
63 
+    for key, tc, size, repeat, payload in _iter_klv(strm_payload):
64 
+        if key == b"SCAL":
65 
+            scal = _parse_scal(tc, payload)
66 
+        elif key == b"GPSF":
67 
+            try:
68 
+                gpsf = struct.unpack(">I", payload[:4])[0]
69 
+            except struct.error:
70 
+                pass
71 
+        elif key == b"GPSP":
72 
+            try:
73 
+                gpsp = struct.unpack(">H", payload[:2])[0]
74 
+            except struct.error:
75 
+                pass
76 
+        elif key == b"GPS5":
77 
+            if scal is None or gpsf < 2 or gpsp > 500 or size < 20:
78 
+                continue
79 
+            s_lat = scal[0] if len(scal) > 0 else 1
80 
+            s_lon = scal[1] if len(scal) > 1 else s_lat
81 
+            s_alt = scal[2] if len(scal) > 2 else 1
82 
+            for i in range(repeat):
83 
+                off = i * size
84 
+                try:
85 
+                    lat_r, lon_r, alt_r = struct.unpack(">iii", payload[off:off + 12])
86 
+                except struct.error:
87 
+                    break
88 
+                lat = lat_r / s_lat
89 
+                lon = lon_r / s_lon
90 
+                alt = alt_r / s_alt
91 
+                if abs(lat) > 0.001 and abs(lon) > 0.001:
92 
+                    return lat, lon, alt
93 
+    return None
94 
+
95 
+
96 
+def parse_gpmf(data):
97 
+    """
98 
+    Walk all DEVC blocks in a raw GPMF binary and return the first valid GPS fix.
99 
+    Returns (lat, lon, alt) or None.
100 
+    """
101 
+    for key, tc, size, repeat, devc_payload in _iter_klv(data):
102 
+        if key != b"DEVC":
103 
+            continue
104 
+        for skey, stc, ssize, srepeat, strm_payload in _iter_klv(devc_payload):
105 
+            if skey != b"STRM":
106 
+                continue
107 
+            result = _first_gps5_fix(strm_payload)
108 
+            if result:
109 
+                return result
110 
+    return None
111 
+
112 
+
113 
+def extract_gpmf_stream(mp4_path):
114 
+    """
115 
+    Use ffprobe to find the GPMF stream index, then extract it with ffmpeg.
116 
+    Returns the binary data or None.
117 
+    """
118 
+    try:
119 
+        probe = subprocess.run(
120 
+            ["ffprobe", "-v", "quiet", "-show_streams", "-print_format", "json", mp4_path],
121 
+            capture_output=True, timeout=30
122 
+        )
123 
+        import json
124 
+        streams = json.loads(probe.stdout).get("streams", [])
125 
+        gpmd_idx = next(
126 
+            (s["index"] for s in streams if s.get("codec_tag_string") == "gpmd"),
127 
+            None
128 
+        )
129 
+        if gpmd_idx is None:
130 
+            return None
131 
+    except Exception:
132 
+        return None
133 
+
134 
+    with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as tmp:
135 
+        tmp_path = tmp.name
136 
+
137 
+    try:
138 
+        subprocess.run(
139 
+            ["ffmpeg", "-y", "-v", "quiet", "-i", mp4_path,
140 
+             "-map", f"0:{gpmd_idx}", "-c", "copy", "-f", "rawvideo", tmp_path],
141 
+            check=True, timeout=60
142 
+        )
143 
+        with open(tmp_path, "rb") as f:
144 
+            return f.read()
145 
+    except Exception:
146 
+        return None
147 
+    finally:
148 
+        try:
149 
+            os.unlink(tmp_path)
150 
+        except OSError:
151 
+            pass
152 
+
153 
+
154 
+def main():
155 
+    if len(sys.argv) != 2:
156 
+        print(f"Usage: {sys.argv[0]} <file.bin|file.mp4>", file=sys.stderr)
157 
+        sys.exit(1)
158 
+
159 
+    path = sys.argv[1]
160 
+    ext = os.path.splitext(path)[1].lower()
161 
+
162 
+    if ext in (".mp4", ".lrv", ".mov"):
163 
+        data = extract_gpmf_stream(path)
164 
+    else:
165 
+        try:
166 
+            with open(path, "rb") as f:
167 
+                data = f.read()
168 
+        except OSError as e:
169 
+            print(f"Error reading {path}: {e}", file=sys.stderr)
170 
+            sys.exit(1)
171 
+
172 
+    if not data:
173 
+        print("No GPMF data found", file=sys.stderr)
174 
+        sys.exit(1)
175 
+
176 
+    result = parse_gpmf(data)
177 
+    if result is None:
178 
+        print("No valid GPS fix found", file=sys.stderr)
179 
+        sys.exit(1)
180 
+
181 
+    lat, lon, alt = result
182 
+    print(f"{lat:.7f} {lon:.7f} {alt:.3f}")
183 
+
184 
+
185 
+if __name__ == "__main__":
186 
+    main()
+93 -1
media-importer.sh
View 
@@ -25,7 +25,10 @@ DRY_RUN=0
25 25 
 VERBOSE=0
26 26 
 CLEANUP_EMPTY_DIRS=1
27 27 
 CLEANUP_MEDIA_SIDECARS=1  # when 1, delete device sidecars (GoPro THM/LRV, Garmin GLV) after successful import
28 
+SYNC_GPS=0              # when 1, embed first GPS fix from GPMF stream into destination file EXIF
28 29 
 CONFLICT_APPLY_ALL=""  # suffix|skip after an interactive "all similar" choice
30 
+
31 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
29 32 
 RESOLVED_DESTINATION_PATH=""
30 33 
 RESERVED_DESTINATION_PATHS=()
31 34 
 
@@ -113,6 +116,7 @@ Options:
113 116 
     --unattended                 Never prompt; resolve destination conflicts with numeric suffixes
114 117 
     --collect-unsortable         Put files without dates into DEST/unsortable
115 118 
     --keep-sidecars              Keep device sidecar files (default: delete after import)
119 
+    --sync-gps                   Embed first valid GPS fix from GPMF stream into destination EXIF
116 120 
     --keep-empty-dirs            Keep empty directories after processing
117 121 
     --dry-run                    Show actions without changing files
118 122 
     -v, --verbose                Verbose output
 
@@ -1088,6 +1092,59 @@ find_source_files() {
1088 1092 
     fi
1089 1093 
 }
1090 1094
1095 
+# Return the companion LRV path for a GoPro source file, or empty string.
1096 
+# GoPro naming: GX/GH/GP chapter files have a matching GL*.LRV low-res proxy.
1097 
+find_companion_lrv() {
1098 
+    local file="$1"
1099 
+    local filename dir stem num
1100 
+    filename=$(basename "$file")
1101 
+    dir=$(dirname "$file")
1102 
+    if [[ "$filename" =~ ^G[HPX]([0-9]{6})\.[Mm][Pp]4$ ]]; then
1103 
+        num="${BASH_REMATCH[1]}"
1104 
+        local lrv
1105 
+        for lrv in "$dir/GL${num}.LRV" "$dir/GL${num}.lrv"; do
1106 
+            [[ -f "$lrv" ]] && echo "$lrv" && return 0
1107 
+        done
1108 
+    fi
1109 
+    return 1
1110 
+}
1111 
+
1112 
+# Extract first valid GPS fix from a GoPro video or LRV file.
1113 
+# Uses ffmpeg to pull the GPMF data stream (fast: reads index only, not video frames),
1114 
+# then parses it with the bundled Python script.
1115 
+# Outputs: "lat lon alt" in decimal degrees / metres.  Exit 1 if no fix found.
1116 
+extract_gps_first_fix() {
1117 
+    local file="$1"
1118 
+    local py="$SCRIPT_DIR/extract_gpmf_gps.py"
1119 
+    if [[ ! -f "$py" ]]; then
1120 
+        return 1
1121 
+    fi
1122 
+    python3 "$py" "$file" 2>/dev/null
1123 
+}
1124 
+
1125 
+# Write GPS coordinates into a file using exiftool QuickTime GPS tags.
1126 
+embed_gps_into_file() {
1127 
+    local file="$1"
1128 
+    local lat="$2" lon="$3" alt="$4"
1129 
+
1130 
+    local ref_lat="N" ref_lon="E" ref_alt="0"
1131 
+    if awk "BEGIN{exit !($lat < 0)}" 2>/dev/null; then
1132 
+        ref_lat="S"; lat="${lat#-}"
1133 
+    fi
1134 
+    if awk "BEGIN{exit !($lon < 0)}" 2>/dev/null; then
1135 
+        ref_lon="W"; lon="${lon#-}"
1136 
+    fi
1137 
+    if awk "BEGIN{exit !($alt < 0)}" 2>/dev/null; then
1138 
+        ref_alt="1"   # below sea level
1139 
+    fi
1140 
+
1141 
+    exiftool -overwrite_original \
1142 
+        "-GPSLatitude=$lat"   "-GPSLatitudeRef=$ref_lat" \
1143 
+        "-GPSLongitude=$lon"  "-GPSLongitudeRef=$ref_lon" \
1144 
+        "-GPSAltitude=$alt"   "-GPSAltitudeRef=$ref_alt" \
1145 
+        "$file" >/dev/null 2>&1
1146 
+}
1147 
+
1091 1148 
 cleanup_media_sidecars() {
1092 1149 
     local file="$1"
1093 1150 
     local dir base stem ext sidecar_ext sidecar
 
@@ -1238,13 +1295,28 @@ process_file() {
1238 1295 
         return 0
1239 1296 
     fi
1240 1297
1298 
+    # Pre-extract GPS from source before the move (source may be on faster local media).
1299 
+    # LRV companion preferred: same GPMF data, smaller file, faster extraction.
1300 
+    local pre_gps_lat="" pre_gps_lon="" pre_gps_alt=""
1301 
+    if [[ $SYNC_GPS -eq 1 ]]; then
1302 
+        local lrv_companion
1303 
+        lrv_companion=$(find_companion_lrv "$file")
1304 
+        local gps_src="${lrv_companion:-$file}"
1305 
+        read -r pre_gps_lat pre_gps_lon pre_gps_alt < <(extract_gps_first_fix "$gps_src")
1306 
+        if [[ -n "$pre_gps_lat" ]]; then
1307 
+            log_message "GPS: first fix at ($pre_gps_lat, $pre_gps_lon, ${pre_gps_alt}m) from $(basename "$gps_src")" "INFO"
1308 
+        else
1309 
+            log_message "GPS: no valid fix found in $(basename "$gps_src")" "INFO"
1310 
+        fi
1311 
+    fi
1312 
+
1241 1313 
     # Create destination directory
1242 1314 
     if ! mkdir -p "$dest_dir"; then
1243 1315 
         log_message "Could not create directory: $dest_dir" "ERROR"
1244 1316 
         ERROR_FILES=$((ERROR_FILES + 1))
1245 1317 
         return 1
1246 1318 
     fi
1247 
-
1319 
+
1248 1320 
     local sync_metadata_after_copy=0
1249 1321 
     local verification_date="$date_str"
1250 1322 
     if should_sync_imported_metadata "$original_basename" "$date_source"; then
 
@@ -1266,6 +1338,11 @@ process_file() {
1266 1338 
                 fi
1267 1339 
             fi
1268 1340 
             log_message "Copied: $file -> $dest_path" "SUCCESS"
1341 
+            if [[ $SYNC_GPS -eq 1 && -n "$pre_gps_lat" ]]; then
1342 
+                embed_gps_into_file "$dest_path" "$pre_gps_lat" "$pre_gps_lon" "$pre_gps_alt" \
1343 
+                    && log_message "GPS: embedded into $dest_path" "INFO" \
1344 
+                    || log_message "GPS: failed to embed into $dest_path" "WARNING"
1345 
+            fi
1269 1346 
             if [[ $CLEANUP_MEDIA_SIDECARS -eq 1 ]]; then
1270 1347 
                 cleanup_media_sidecars "$file"
1271 1348 
             fi
 
@@ -1295,6 +1372,11 @@ process_file() {
1295 1372 
                     return 1
1296 1373 
                 fi
1297 1374 
                 log_message "Moved: $file -> $dest_path" "SUCCESS"
1375 
+            if [[ $SYNC_GPS -eq 1 && -n "$pre_gps_lat" ]]; then
1376 
+                embed_gps_into_file "$dest_path" "$pre_gps_lat" "$pre_gps_lon" "$pre_gps_alt" \
1377 
+                    && log_message "GPS: embedded into $dest_path" "INFO" \
1378 
+                    || log_message "GPS: failed to embed into $dest_path" "WARNING"
1379 
+            fi
1298 1380 
             if [[ $CLEANUP_MEDIA_SIDECARS -eq 1 ]]; then
1299 1381 
                 cleanup_media_sidecars "$file"
1300 1382 
             fi
 
@@ -1308,6 +1390,11 @@ process_file() {
1308 1390 
             fi
1309 1391 
         elif verified_move_file "$file" "$dest_path" "$date_str"; then
1310 1392 
             log_message "Moved: $file -> $dest_path" "SUCCESS"
1393 
+            if [[ $SYNC_GPS -eq 1 && -n "$pre_gps_lat" ]]; then
1394 
+                embed_gps_into_file "$dest_path" "$pre_gps_lat" "$pre_gps_lon" "$pre_gps_alt" \
1395 
+                    && log_message "GPS: embedded into $dest_path" "INFO" \
1396 
+                    || log_message "GPS: failed to embed into $dest_path" "WARNING"
1397 
+            fi
1311 1398 
             if [[ $CLEANUP_MEDIA_SIDECARS -eq 1 ]]; then
1312 1399 
                 cleanup_media_sidecars "$file"
1313 1400 
             fi
 
@@ -1457,6 +1544,10 @@ while [[ $# -gt 0 ]]; do
1457 1544 
             CLEANUP_MEDIA_SIDECARS=0
1458 1545 
             shift
1459 1546 
             ;;
1547 
+        --sync-gps)
1548 
+            SYNC_GPS=1
1549 
+            shift
1550 
+            ;;
1460 1551 
         --dry-run)
1461 1552 
             DRY_RUN=1
1462 1553 
             shift
 
@@ -1542,6 +1633,7 @@ echo "  Verify mode:         $VERIFY_MODE"
1542 1633 
 echo "  Date source:         $DATE_SOURCE"
1543 1634 
 echo "  Sync metadata:       $([ $SYNC_METADATA -eq 1 ] && echo "Yes" || echo "No")"
1544 1635 
 echo "  QuickTime UTC:       $([ $QUICKTIME_UTC -eq 1 ] && echo "Yes" || echo "No (local time assumed)")"
1636 
+echo "  Sync GPS:            $([ $SYNC_GPS -eq 1 ] && echo "Yes" || echo "No")"
1545 1637 
 echo "  Unattended:          $([ $UNATTENDED -eq 1 ] && echo "Yes" || echo "No")"
1546 1638 
 echo "  Dry run:             $([ $DRY_RUN -eq 1 ] && echo "Yes" || echo "No")"
1547 1639 
 echo "  Keep empty dirs:     $([ $CLEANUP_EMPTY_DIRS -eq 0 ] && echo "Yes" || echo "No")"
 

              

          

    

  


  








  

    

      Gitprep Version v2.6.2
      build on Mojolicious
    


      

        DBViewer