Read archive records from visibility ranges · 37ccabd

Read archive records from visibility ranges
Browse files

bogdan committed 2 weeks ago

main

1 parent 08a5014

commit 37ccabd

Showing 5 changed files with 131 additions and 67 deletions

+5 -4

HealthProbe/Doc/02-architecture/Database-Design.md

@@ -347,7 +347,8 @@ ON sample_visibility_ranges(first_observation_id, last_observation_id);
 
 Range convention:
 - `last_observation_id IS NULL` means still visible at the latest verified observation for that type;
-- closed ranges represent observations where the sample/version was visible;
+- `last_observation_id` stores the observation that closed the range and is therefore an exclusive end;
+- closed samples/versions are visible when `first_observation_id <= target.id` and `last_observation_id > target.id`;
 - deleted-object evidence should create an event even when full payload is not available.
 
 ### 5.7 Relationships
@@ -582,7 +583,7 @@ JOIN sample_versions sv ON sv.id = r.version_id
 JOIN observations target ON target.id = :observation_id
 WHERE s.sample_type_id = :sample_type_id
   AND r.first_observation_id <= target.id
-  AND (r.last_observation_id IS NULL OR r.last_observation_id >= target.id)
+  AND (r.last_observation_id IS NULL OR r.last_observation_id > target.id)
 ORDER BY sv.start_date, s.strict_fingerprint
 LIMIT :limit OFFSET :offset;
 ```
@@ -598,13 +599,13 @@ CREATE TEMP TABLE prev_visible AS
 SELECT r.sample_id, r.version_id
 FROM sample_visibility_ranges r
 WHERE r.first_observation_id <= :previous
-  AND (r.last_observation_id IS NULL OR r.last_observation_id >= :previous);
+  AND (r.last_observation_id IS NULL OR r.last_observation_id > :previous);
 
 CREATE TEMP TABLE curr_visible AS
 SELECT r.sample_id, r.version_id
 FROM sample_visibility_ranges r
 WHERE r.first_observation_id <= :current
-  AND (r.last_observation_id IS NULL OR r.last_observation_id >= :current);
+  AND (r.last_observation_id IS NULL OR r.last_observation_id > :current);
 
 CREATE INDEX temp_prev_sample ON prev_visible(sample_id);
 CREATE INDEX temp_curr_sample ON curr_visible(sample_id);


+8 -9

HealthProbe/Doc/04-project/IMPLEMENTATION_STATUS.md

View

@@ -25,7 +25,7 @@ There are no real deployments, only test installations. Existing prototype datab
 |------|----------------|--------------------|
 | Product docs | Updated | Keep `HealthProbe/Doc/README.md` as canonical index |
 | HealthKit capture | Prototype exists | Adapt capture to write differential SQLite observations first |
-| SQLite archive | Archive v2 schema, differential write path, daily aggregate rebuilds, and integrity report are partially implemented; legacy read table still active | Add SQL reads, open/schema-version tests, idempotency tests, then retire `archive_samples` |
+| SQLite archive | Archive v2 schema, differential write path, daily aggregate rebuilds, integrity report, and v2 record reads are partially implemented; legacy write mirror still exists | Add diff SQL, open/schema-version tests, idempotency tests, then retire `archive_samples` |
 | Core Data cache | Not implemented | Add rebuildable cache for expensive counts, summaries, report metadata, UI state |
 | SwiftData cache | Exists | Treat as disposable prototype data; reset/ignore during v2 transition |
 | UI | Prototype exists | Reframe screens around observations, diffs, export, archive status |
@@ -38,15 +38,14 @@ There are no real deployments, only test installations. Existing prototype datab
 
 Detailed checkable milestones live in [`Refactoring-Plan.md`](Refactoring-Plan.md).
 
-1. Finish differential write path hardening with stricter retry/idempotency tests.
+1. Finish differential write path hardening with retry/idempotency tests.
 2. Add SQLite integrity/open/schema-version tests.
-3. Move archive reads from `archive_samples` to SQL over visibility ranges and sample versions.
-4. Move large diffs/counts into SQL queries with indexes/temp tables/paged results.
-5. Add Core Data UI/report cache and rebuild pipeline.
-6. Replace SwiftData UI dependencies with Core Data/cache DTOs.
-7. Update UI language from anomaly/status to observation/diff/export.
-8. Add streaming exports with manifests.
-9. Validate on low-memory/legacy-class devices.
+3. Move large diffs/counts into SQL queries with indexes/temp tables/paged results.
+4. Add Core Data UI/report cache and rebuild pipeline.
+5. Replace SwiftData UI dependencies with Core Data/cache DTOs.
+6. Update UI language from anomaly/status to observation/diff/export.
+7. Add streaming exports with manifests.
+8. Validate on low-memory/legacy-class devices.
 
 ## Known Prototype Mismatches
 


+3 -3

HealthProbe/Doc/04-project/Refactoring-Plan.md

View

@@ -143,8 +143,8 @@ Acceptance:
 **Purpose:** Make the archive useful without RAM-heavy processing.
 
 Checklist:
-- [ ] Implement point-in-time visible-record query.
-- [ ] Implement paged record table query.
+- [x] Implement point-in-time visible-record query.
+- [x] Implement paged record table query.
 - [ ] Implement appeared query between observations.
 - [ ] Implement disappeared query between observations.
 - [ ] Implement representationChanged query between observations.
@@ -155,7 +155,7 @@ Checklist:
 - [ ] Add query timing/memory tests on synthetic large datasets.
 
 Acceptance:
-- [ ] Observation T can be reconstructed from ranges/events.
+- [x] Observation T can be reconstructed from ranges/events.
 - [ ] Large diff returns counts and first page without loading all rows.
 - [ ] Query results are deterministic and ordered.
 - [ ] Consolidation evidence includes count, aggregate, coverage, density, and uncertainty data.


+4 -1

HealthProbe/Services/Protocols/HealthArchiveStore.swift

View

@@ -33,6 +33,7 @@ struct HealthArchiveIntegrityReport: Equatable, Sendable {
 }
 
 struct HealthArchiveRecordRequest: Equatable, Sendable {
+    let visibleAtObservationID: Int64?
     let sampleTypeIdentifier: String?
     let fingerprints: Set<String>
     let disappearedOnly: Bool
@@ -41,7 +42,8 @@ struct HealthArchiveRecordRequest: Equatable, Sendable {
     let afterCursor: RecordCursor?
     let limit: Int?
 
-    init(
+    nonisolated init(
+        visibleAtObservationID: Int64? = nil,
         sampleTypeIdentifier: String? = nil,
         fingerprints: Set<String> = [],
         disappearedOnly: Bool = false,
@@ -50,6 +52,7 @@ struct HealthArchiveRecordRequest: Equatable, Sendable {
         afterCursor: RecordCursor? = nil,
         limit: Int? = nil
     ) {
+        self.visibleAtObservationID = visibleAtObservationID
         self.sampleTypeIdentifier = sampleTypeIdentifier
         self.fingerprints = fingerprints
         self.disappearedOnly = disappearedOnly


+111 -50

HealthProbe/Services/SQLiteHealthArchiveStore.swift

View

@@ -151,38 +151,92 @@ actor SQLiteHealthArchiveStore: HealthArchiveStore {
 
         var clauses: [String] = []
         if request.sampleTypeIdentifier != nil {
-            clauses.append("type_identifier = ?")
+            clauses.append("t.type_identifier = ?")
         }
         if !request.fingerprints.isEmpty {
-            clauses.append("strict_fingerprint IN (\(Array(repeating: "?", count: request.fingerprints.count).joined(separator: ",")))")
+            clauses.append("s.strict_fingerprint IN (\(Array(repeating: "?", count: request.fingerprints.count).joined(separator: ",")))")
         }
         if request.disappearedOnly {
-            clauses.append("disappeared_at IS NOT NULL")
+            clauses.append("rr.last_observation_id IS NOT NULL AND es.disappeared_at IS NOT NULL")
         }
         if request.firstSeenAfter != nil {
-            clauses.append("first_seen_at >= ?")
+            clauses.append("s.first_seen_at >= ?")
         }
         if request.firstSeenBefore != nil {
-            clauses.append("first_seen_at <= ?")
+            clauses.append("s.first_seen_at <= ?")
         }
         if request.afterCursor != nil {
-            clauses.append("(start_date > ? OR (start_date = ? AND strict_fingerprint > ?))")
+            clauses.append("(v.start_date > ? OR (v.start_date = ? AND s.strict_fingerprint > ?))")
         }
         let whereClause = clauses.isEmpty ? "" : "WHERE \(clauses.joined(separator: " AND "))"
         let limitClause = request.limit.map { "LIMIT \(max($0, 0))" } ?? ""
         let sql = """
-        SELECT sample_uuid_hash, type_identifier, strict_fingerprint, semantic_fingerprint,
-               start_date, end_date, first_seen_at, last_seen_at, last_verified_at, disappeared_at,
-               value_kind, value, unit, category_value, workout_activity_type, duration_seconds,
-               source_name, source_bundle_identifier, device_name
-        FROM archive_samples
+        WITH selected_ranges AS (
+            SELECT
+                r.sample_id, r.version_id, r.first_observation_id, r.last_observation_id,
+                r.first_seen_at, r.last_seen_at,
+                ROW_NUMBER() OVER (
+                    PARTITION BY r.sample_id
+                    ORDER BY
+                        CASE
+                            WHEN ? IS NOT NULL THEN 0
+                            WHEN r.last_observation_id IS NULL THEN 0
+                            ELSE 1
+                        END,
+                        COALESCE(r.last_observation_id, 9223372036854775807) DESC,
+                        r.first_observation_id DESC
+                ) AS record_rank
+            FROM sample_visibility_ranges r
+            WHERE (? IS NULL OR (
+                r.first_observation_id <= ?
+                AND (r.last_observation_id IS NULL OR r.last_observation_id > ?)
+            ))
+        ),
+        record_ranges AS (
+            SELECT *
+            FROM selected_ranges
+            WHERE record_rank = 1
+        ),
+        event_summary AS (
+            SELECT
+                sample_id,
+                MAX(CASE WHEN event_kind != 'disappeared' THEN observed_at END) AS last_seen_at,
+                MAX(observed_at) AS last_verified_at,
+                MAX(CASE WHEN event_kind = 'disappeared' THEN observed_at END) AS disappeared_at
+            FROM sample_observation_events
+            GROUP BY sample_id
+        )
+        SELECT
+               COALESCE(s.sample_uuid_hash, s.strict_fingerprint) AS record_id,
+               t.type_identifier, s.strict_fingerprint, s.semantic_fingerprint, s.sample_uuid_hash,
+               v.start_date, v.end_date, s.first_seen_at,
+               COALESCE(es.last_seen_at, rr.first_seen_at) AS last_seen_at,
+               es.last_verified_at,
+               CASE WHEN rr.last_observation_id IS NULL THEN NULL ELSE es.disappeared_at END AS disappeared_at,
+               v.value_kind, v.numeric_value, v.unit, v.category_value, v.workout_activity_type, v.duration_seconds,
+               src.bundle_identifier
+        FROM record_ranges rr
+        JOIN samples s ON s.id = rr.sample_id
+        JOIN sample_types t ON t.id = s.sample_type_id
+        JOIN sample_versions v ON v.id = rr.version_id
+        LEFT JOIN source_revisions sr ON sr.id = v.source_revision_id
+        LEFT JOIN sources src ON src.id = sr.source_id
+        LEFT JOIN event_summary es ON es.sample_id = s.id
         \(whereClause)
-        ORDER BY start_date ASC, strict_fingerprint ASC
+        ORDER BY v.start_date ASC, s.strict_fingerprint ASC
         \(limitClause)
         """
 
         return try withStatement(sql, db: db) { statement in
             var index: Int32 = 1
+            bindInt64(request.visibleAtObservationID, to: index, in: statement)
+            index += 1
+            bindInt64(request.visibleAtObservationID, to: index, in: statement)
+            index += 1
+            bindInt64(request.visibleAtObservationID, to: index, in: statement)
+            index += 1
+            bindInt64(request.visibleAtObservationID, to: index, in: statement)
+            index += 1
             if let typeIdentifier = request.sampleTypeIdentifier {
                 bindText(typeIdentifier, to: index, in: statement)
                 index += 1
@@ -192,17 +246,17 @@ actor SQLiteHealthArchiveStore: HealthArchiveStore {
                 index += 1
             }
             if let firstSeenAfter = request.firstSeenAfter {
-                sqlite3_bind_double(statement, index, firstSeenAfter.timeIntervalSinceReferenceDate)
+                sqlite3_bind_double(statement, index, firstSeenAfter.timeIntervalSince1970)
                 index += 1
             }
             if let firstSeenBefore = request.firstSeenBefore {
-                sqlite3_bind_double(statement, index, firstSeenBefore.timeIntervalSinceReferenceDate)
+                sqlite3_bind_double(statement, index, firstSeenBefore.timeIntervalSince1970)
                 index += 1
             }
             if let cursor = request.afterCursor {
-                sqlite3_bind_double(statement, index, cursor.startDate.timeIntervalSinceReferenceDate)
+                sqlite3_bind_double(statement, index, cursor.startDate.timeIntervalSince1970)
                 index += 1
-                sqlite3_bind_double(statement, index, cursor.startDate.timeIntervalSinceReferenceDate)
+                sqlite3_bind_double(statement, index, cursor.startDate.timeIntervalSince1970)
                 index += 1
                 bindText(cursor.strictFingerprint, to: index, in: statement)
                 index += 1
@@ -215,22 +269,22 @@ actor SQLiteHealthArchiveStore: HealthArchiveStore {
                     sampleTypeIdentifier: columnText(statement, 1) ?? "",
                     strictFingerprint: columnText(statement, 2) ?? "",
                     semanticFingerprint: columnText(statement, 3),
-                    healthKitUUIDHash: columnText(statement, 0),
-                    startDate: columnDate(statement, 4) ?? Date(timeIntervalSinceReferenceDate: 0),
-                    endDate: columnDate(statement, 5) ?? Date(timeIntervalSinceReferenceDate: 0),
-                    firstSeenAt: columnDate(statement, 6) ?? Date(timeIntervalSinceReferenceDate: 0),
-                    lastSeenAt: columnDate(statement, 7),
-                    lastVerifiedAt: columnDate(statement, 8),
-                    disappearedAt: columnDate(statement, 9),
-                    valueKind: columnText(statement, 10),
-                    value: columnDouble(statement, 11),
-                    unit: columnText(statement, 12),
-                    categoryValue: columnInt(statement, 13),
-                    workoutActivityType: columnInt(statement, 14),
-                    durationSeconds: columnDouble(statement, 15),
-                    sourceName: columnText(statement, 16),
+                    healthKitUUIDHash: columnText(statement, 4),
+                    startDate: columnUnixDate(statement, 5) ?? Date(timeIntervalSince1970: 0),
+                    endDate: columnUnixDate(statement, 6) ?? Date(timeIntervalSince1970: 0),
+                    firstSeenAt: columnUnixDate(statement, 7) ?? Date(timeIntervalSince1970: 0),
+                    lastSeenAt: columnUnixDate(statement, 8),
+                    lastVerifiedAt: columnUnixDate(statement, 9),
+                    disappearedAt: columnUnixDate(statement, 10),
+                    valueKind: columnText(statement, 11),
+                    value: columnDouble(statement, 12),
+                    unit: columnText(statement, 13),
+                    categoryValue: columnInt(statement, 14),
+                    workoutActivityType: columnInt(statement, 15),
+                    durationSeconds: columnDouble(statement, 16),
+                    sourceName: nil,
                     sourceBundleIdentifier: columnText(statement, 17),
-                    deviceName: columnText(statement, 18)
+                    deviceName: nil
                 ))
             }
             return records
@@ -997,13 +1051,14 @@ actor SQLiteHealthArchiveStore: HealthArchiveStore {
               AND (bundle_identifier = ? OR (bundle_identifier IS NULL AND ? IS NULL))
             LIMIT 1
             """,
-            db: db
-        ) { statement in
-            bindText(sourceNameHash, to: 1, in: statement)
-            bindText(sourceNameHash, to: 2, in: statement)
-            bindText(bundleIdentifier, to: 3, in: statement)
-            bindText(bundleIdentifier, to: 4, in: statement)
-        } {
+            db: db,
+            bind: { statement in
+                bindText(sourceNameHash, to: 1, in: statement)
+                bindText(sourceNameHash, to: 2, in: statement)
+                bindText(bundleIdentifier, to: 3, in: statement)
+                bindText(bundleIdentifier, to: 4, in: statement)
+            }
+        ) {
             return existing
         }
         try withStatement("INSERT INTO sources (source_name_hash, bundle_identifier) VALUES (?, ?)", db: db) { statement in
@@ -1032,17 +1087,18 @@ actor SQLiteHealthArchiveStore: HealthArchiveStore {
               AND (model = ? OR (model IS NULL AND ? IS NULL))
             LIMIT 1
             """,
-            db: db
-        ) { statement in
-            bindText(deviceHash, to: 1, in: statement)
-            bindText(deviceHash, to: 2, in: statement)
-            bindText(localIdentifierHash, to: 3, in: statement)
-            bindText(localIdentifierHash, to: 4, in: statement)
-            bindText(udiHash, to: 5, in: statement)
-            bindText(udiHash, to: 6, in: statement)
-            bindText(row.deviceModel, to: 7, in: statement)
-            bindText(row.deviceModel, to: 8, in: statement)
-        } {
+            db: db,
+            bind: { statement in
+                bindText(deviceHash, to: 1, in: statement)
+                bindText(deviceHash, to: 2, in: statement)
+                bindText(localIdentifierHash, to: 3, in: statement)
+                bindText(localIdentifierHash, to: 4, in: statement)
+                bindText(udiHash, to: 5, in: statement)
+                bindText(udiHash, to: 6, in: statement)
+                bindText(row.deviceModel, to: 7, in: statement)
+                bindText(row.deviceModel, to: 8, in: statement)
+            }
+        ) {
             return existing
         }
 
@@ -1818,7 +1874,7 @@ private struct ArchiveV2DailyAggregateRow {
     let valueMax: Double?
     let sourceRevisionID: Int64?
 
-    func hashParts(observationID: Int64, sampleTypeID: Int64) -> [String?] {
+    nonisolated func hashParts(observationID: Int64, sampleTypeID: Int64) -> [String?] {
         [
             String(observationID),
             String(sampleTypeID),
@@ -1893,6 +1949,11 @@ nonisolated private func columnDate(_ statement: OpaquePointer?, _ index: Int32)
     return Date(timeIntervalSinceReferenceDate: sqlite3_column_double(statement, index))
 }
 
+nonisolated private func columnUnixDate(_ statement: OpaquePointer?, _ index: Int32) -> Date? {
+    guard sqlite3_column_type(statement, index) != SQLITE_NULL else { return nil }
+    return Date(timeIntervalSince1970: sqlite3_column_double(statement, index))
+}
+
 nonisolated private func columnDouble(_ statement: OpaquePointer?, _ index: Int32) -> Double? {
     guard sqlite3_column_type(statement, index) != SQLITE_NULL else { return nil }
     return sqlite3_column_double(statement, index)


	@@ -347,7 +347,8 @@ ON sample_visibility_ranges(first_observation_id, last_observation_id);
347	347
348	348	Range convention:
349	349	- `last_observation_id IS NULL` means still visible at the latest verified observation for that type;
350		-- closed ranges represent observations where the sample/version was visible;
	350	+- `last_observation_id` stores the observation that closed the range and is therefore an exclusive end;
	351	+- closed samples/versions are visible when `first_observation_id <= target.id` and `last_observation_id > target.id`;
351	352	- deleted-object evidence should create an event even when full payload is not available.
352	353
353	354	### 5.7 Relationships
	@@ -582,7 +583,7 @@ JOIN sample_versions sv ON sv.id = r.version_id
582	583	JOIN observations target ON target.id = :observation_id
583	584	WHERE s.sample_type_id = :sample_type_id
584	585	AND r.first_observation_id <= target.id
585		- AND (r.last_observation_id IS NULL OR r.last_observation_id >= target.id)
	586	+ AND (r.last_observation_id IS NULL OR r.last_observation_id > target.id)
586	587	ORDER BY sv.start_date, s.strict_fingerprint
587	588	LIMIT :limit OFFSET :offset;
588	589	```
	@@ -598,13 +599,13 @@ CREATE TEMP TABLE prev_visible AS
598	599	SELECT r.sample_id, r.version_id
599	600	FROM sample_visibility_ranges r
600	601	WHERE r.first_observation_id <= :previous
601		- AND (r.last_observation_id IS NULL OR r.last_observation_id >= :previous);
	602	+ AND (r.last_observation_id IS NULL OR r.last_observation_id > :previous);
602	603
603	604	CREATE TEMP TABLE curr_visible AS
604	605	SELECT r.sample_id, r.version_id
605	606	FROM sample_visibility_ranges r
606	607	WHERE r.first_observation_id <= :current
607		- AND (r.last_observation_id IS NULL OR r.last_observation_id >= :current);
	608	+ AND (r.last_observation_id IS NULL OR r.last_observation_id > :current);
608	609
609	610	CREATE INDEX temp_prev_sample ON prev_visible(sample_id);
610	611	CREATE INDEX temp_curr_sample ON curr_visible(sample_id);

	@@ -143,8 +143,8 @@ Acceptance:
143	143	Purpose: Make the archive useful without RAM-heavy processing.
144	144
145	145	Checklist:
146		-- [ ] Implement point-in-time visible-record query.
147		-- [ ] Implement paged record table query.
	146	+- [x] Implement point-in-time visible-record query.
	147	+- [x] Implement paged record table query.
148	148	- [ ] Implement appeared query between observations.
149	149	- [ ] Implement disappeared query between observations.
150	150	- [ ] Implement representationChanged query between observations.
	@@ -155,7 +155,7 @@ Checklist:
155	155	- [ ] Add query timing/memory tests on synthetic large datasets.
156	156
157	157	Acceptance:
158		-- [ ] Observation T can be reconstructed from ranges/events.
	158	+- [x] Observation T can be reconstructed from ranges/events.
159	159	- [ ] Large diff returns counts and first page without loading all rows.
160	160	- [ ] Query results are deterministic and ordered.
161	161	- [ ] Consolidation evidence includes count, aggregate, coverage, density, and uncertainty data.

	@@ -25,7 +25,7 @@ There are no real deployments, only test installations. Existing prototype datab
25	25	\|------\|----------------\|--------------------\|
26	26	\| Product docs \| Updated \| Keep `HealthProbe/Doc/README.md` as canonical index \|
27	27	\| HealthKit capture \| Prototype exists \| Adapt capture to write differential SQLite observations first \|
28		-\| SQLite archive \| Archive v2 schema, differential write path, daily aggregate rebuilds, and integrity report are partially implemented; legacy read table still active \| Add SQL reads, open/schema-version tests, idempotency tests, then retire `archive_samples` \|
	28	+\| SQLite archive \| Archive v2 schema, differential write path, daily aggregate rebuilds, integrity report, and v2 record reads are partially implemented; legacy write mirror still exists \| Add diff SQL, open/schema-version tests, idempotency tests, then retire `archive_samples` \|
29	29	\| Core Data cache \| Not implemented \| Add rebuildable cache for expensive counts, summaries, report metadata, UI state \|
30	30	\| SwiftData cache \| Exists \| Treat as disposable prototype data; reset/ignore during v2 transition \|
31	31	\| UI \| Prototype exists \| Reframe screens around observations, diffs, export, archive status \|
	@@ -38,15 +38,14 @@ There are no real deployments, only test installations. Existing prototype datab
38	38
39	39	Detailed checkable milestones live in [`Refactoring-Plan.md`](Refactoring-Plan.md).
40	40
41		-1. Finish differential write path hardening with stricter retry/idempotency tests.
	41	+1. Finish differential write path hardening with retry/idempotency tests.
42	42	2. Add SQLite integrity/open/schema-version tests.
43		-3. Move archive reads from `archive_samples` to SQL over visibility ranges and sample versions.
44		-4. Move large diffs/counts into SQL queries with indexes/temp tables/paged results.
45		-5. Add Core Data UI/report cache and rebuild pipeline.
46		-6. Replace SwiftData UI dependencies with Core Data/cache DTOs.
47		-7. Update UI language from anomaly/status to observation/diff/export.
48		-8. Add streaming exports with manifests.
49		-9. Validate on low-memory/legacy-class devices.
	43	+3. Move large diffs/counts into SQL queries with indexes/temp tables/paged results.
	44	+4. Add Core Data UI/report cache and rebuild pipeline.
	45	+5. Replace SwiftData UI dependencies with Core Data/cache DTOs.
	46	+6. Update UI language from anomaly/status to observation/diff/export.
	47	+7. Add streaming exports with manifests.
	48	+8. Validate on low-memory/legacy-class devices.
50	49
51	50	## Known Prototype Mismatches
52	51

	@@ -33,6 +33,7 @@ struct HealthArchiveIntegrityReport: Equatable, Sendable {
33	33	}
34	34
35	35	struct HealthArchiveRecordRequest: Equatable, Sendable {
	36	+ let visibleAtObservationID: Int64?
36	37	let sampleTypeIdentifier: String?
37	38	let fingerprints: Set<String>
38	39	let disappearedOnly: Bool
	@@ -41,7 +42,8 @@ struct HealthArchiveRecordRequest: Equatable, Sendable {
41	42	let afterCursor: RecordCursor?
42	43	let limit: Int?
43	44
44		- init(
	45	+ nonisolated init(
	46	+ visibleAtObservationID: Int64? = nil,
45	47	sampleTypeIdentifier: String? = nil,
46	48	fingerprints: Set<String> = [],
47	49	disappearedOnly: Bool = false,
	@@ -50,6 +52,7 @@ struct HealthArchiveRecordRequest: Equatable, Sendable {
50	52	afterCursor: RecordCursor? = nil,
51	53	limit: Int? = nil
52	54	) {
	55	+ self.visibleAtObservationID = visibleAtObservationID
53	56	self.sampleTypeIdentifier = sampleTypeIdentifier
54	57	self.fingerprints = fingerprints
55	58	self.disappearedOnly = disappearedOnly

	@@ -151,38 +151,92 @@ actor SQLiteHealthArchiveStore: HealthArchiveStore {
151	151
152	152	var clauses: [String] = []
153	153	if request.sampleTypeIdentifier != nil {
154		- clauses.append("type_identifier = ?")
	154	+ clauses.append("t.type_identifier = ?")
155	155	}
156	156	if !request.fingerprints.isEmpty {
157		- clauses.append("strict_fingerprint IN (\(Array(repeating: "?", count: request.fingerprints.count).joined(separator: ",")))")
	157	+ clauses.append("s.strict_fingerprint IN (\(Array(repeating: "?", count: request.fingerprints.count).joined(separator: ",")))")
158	158	}
159	159	if request.disappearedOnly {
160		- clauses.append("disappeared_at IS NOT NULL")
	160	+ clauses.append("rr.last_observation_id IS NOT NULL AND es.disappeared_at IS NOT NULL")
161	161	}
162	162	if request.firstSeenAfter != nil {
163		- clauses.append("first_seen_at >= ?")
	163	+ clauses.append("s.first_seen_at >= ?")
164	164	}
165	165	if request.firstSeenBefore != nil {
166		- clauses.append("first_seen_at <= ?")
	166	+ clauses.append("s.first_seen_at <= ?")
167	167	}
168	168	if request.afterCursor != nil {
169		- clauses.append("(start_date > ? OR (start_date = ? AND strict_fingerprint > ?))")
	169	+ clauses.append("(v.start_date > ? OR (v.start_date = ? AND s.strict_fingerprint > ?))")
170	170	}
171	171	let whereClause = clauses.isEmpty ? "" : "WHERE \(clauses.joined(separator: " AND "))"
172	172	let limitClause = request.limit.map { "LIMIT \(max($0, 0))" } ?? ""
173	173	let sql = """
174		- SELECT sample_uuid_hash, type_identifier, strict_fingerprint, semantic_fingerprint,
175		- start_date, end_date, first_seen_at, last_seen_at, last_verified_at, disappeared_at,
176		- value_kind, value, unit, category_value, workout_activity_type, duration_seconds,
177		- source_name, source_bundle_identifier, device_name
178		- FROM archive_samples
	174	+ WITH selected_ranges AS (
	175	+ SELECT
	176	+ r.sample_id, r.version_id, r.first_observation_id, r.last_observation_id,
	177	+ r.first_seen_at, r.last_seen_at,
	178	+ ROW_NUMBER() OVER (
	179	+ PARTITION BY r.sample_id
	180	+ ORDER BY
	181	+ CASE
	182	+ WHEN ? IS NOT NULL THEN 0
	183	+ WHEN r.last_observation_id IS NULL THEN 0
	184	+ ELSE 1
	185	+ END,
	186	+ COALESCE(r.last_observation_id, 9223372036854775807) DESC,
	187	+ r.first_observation_id DESC
	188	+ ) AS record_rank
	189	+ FROM sample_visibility_ranges r
	190	+ WHERE (? IS NULL OR (
	191	+ r.first_observation_id <= ?
	192	+ AND (r.last_observation_id IS NULL OR r.last_observation_id > ?)
	193	+ ))
	194	+ ),
	195	+ record_ranges AS (
	196	+ SELECT *
	197	+ FROM selected_ranges
	198	+ WHERE record_rank = 1
	199	+ ),
	200	+ event_summary AS (
	201	+ SELECT
	202	+ sample_id,
	203	+ MAX(CASE WHEN event_kind != 'disappeared' THEN observed_at END) AS last_seen_at,
	204	+ MAX(observed_at) AS last_verified_at,
	205	+ MAX(CASE WHEN event_kind = 'disappeared' THEN observed_at END) AS disappeared_at
	206	+ FROM sample_observation_events
	207	+ GROUP BY sample_id
	208	+ )
	209	+ SELECT
	210	+ COALESCE(s.sample_uuid_hash, s.strict_fingerprint) AS record_id,
	211	+ t.type_identifier, s.strict_fingerprint, s.semantic_fingerprint, s.sample_uuid_hash,
	212	+ v.start_date, v.end_date, s.first_seen_at,
	213	+ COALESCE(es.last_seen_at, rr.first_seen_at) AS last_seen_at,
	214	+ es.last_verified_at,
	215	+ CASE WHEN rr.last_observation_id IS NULL THEN NULL ELSE es.disappeared_at END AS disappeared_at,
	216	+ v.value_kind, v.numeric_value, v.unit, v.category_value, v.workout_activity_type, v.duration_seconds,
	217	+ src.bundle_identifier
	218	+ FROM record_ranges rr
	219	+ JOIN samples s ON s.id = rr.sample_id
	220	+ JOIN sample_types t ON t.id = s.sample_type_id
	221	+ JOIN sample_versions v ON v.id = rr.version_id
	222	+ LEFT JOIN source_revisions sr ON sr.id = v.source_revision_id
	223	+ LEFT JOIN sources src ON src.id = sr.source_id
	224	+ LEFT JOIN event_summary es ON es.sample_id = s.id
179	225	\(whereClause)
180		- ORDER BY start_date ASC, strict_fingerprint ASC
	226	+ ORDER BY v.start_date ASC, s.strict_fingerprint ASC
181	227	\(limitClause)
182	228	"""
183	229
184	230	return try withStatement(sql, db: db) { statement in
185	231	var index: Int32 = 1
	232	+ bindInt64(request.visibleAtObservationID, to: index, in: statement)
	233	+ index += 1
	234	+ bindInt64(request.visibleAtObservationID, to: index, in: statement)
	235	+ index += 1
	236	+ bindInt64(request.visibleAtObservationID, to: index, in: statement)
	237	+ index += 1
	238	+ bindInt64(request.visibleAtObservationID, to: index, in: statement)
	239	+ index += 1
186	240	if let typeIdentifier = request.sampleTypeIdentifier {
187	241	bindText(typeIdentifier, to: index, in: statement)
188	242	index += 1
	@@ -192,17 +246,17 @@ actor SQLiteHealthArchiveStore: HealthArchiveStore {
192	246	index += 1
193	247	}
194	248	if let firstSeenAfter = request.firstSeenAfter {
195		- sqlite3_bind_double(statement, index, firstSeenAfter.timeIntervalSinceReferenceDate)
	249	+ sqlite3_bind_double(statement, index, firstSeenAfter.timeIntervalSince1970)
196	250	index += 1
197	251	}
198	252	if let firstSeenBefore = request.firstSeenBefore {
199		- sqlite3_bind_double(statement, index, firstSeenBefore.timeIntervalSinceReferenceDate)
	253	+ sqlite3_bind_double(statement, index, firstSeenBefore.timeIntervalSince1970)
200	254	index += 1
201	255	}
202	256	if let cursor = request.afterCursor {
203		- sqlite3_bind_double(statement, index, cursor.startDate.timeIntervalSinceReferenceDate)
	257	+ sqlite3_bind_double(statement, index, cursor.startDate.timeIntervalSince1970)
204	258	index += 1
205		- sqlite3_bind_double(statement, index, cursor.startDate.timeIntervalSinceReferenceDate)
	259	+ sqlite3_bind_double(statement, index, cursor.startDate.timeIntervalSince1970)
206	260	index += 1
207	261	bindText(cursor.strictFingerprint, to: index, in: statement)
208	262	index += 1
	@@ -215,22 +269,22 @@ actor SQLiteHealthArchiveStore: HealthArchiveStore {
215	269	sampleTypeIdentifier: columnText(statement, 1) ?? "",
216	270	strictFingerprint: columnText(statement, 2) ?? "",
217	271	semanticFingerprint: columnText(statement, 3),
218		- healthKitUUIDHash: columnText(statement, 0),
219		- startDate: columnDate(statement, 4) ?? Date(timeIntervalSinceReferenceDate: 0),
220		- endDate: columnDate(statement, 5) ?? Date(timeIntervalSinceReferenceDate: 0),
221		- firstSeenAt: columnDate(statement, 6) ?? Date(timeIntervalSinceReferenceDate: 0),
222		- lastSeenAt: columnDate(statement, 7),
223		- lastVerifiedAt: columnDate(statement, 8),
224		- disappearedAt: columnDate(statement, 9),
225		- valueKind: columnText(statement, 10),
226		- value: columnDouble(statement, 11),
227		- unit: columnText(statement, 12),
228		- categoryValue: columnInt(statement, 13),
229		- workoutActivityType: columnInt(statement, 14),
230		- durationSeconds: columnDouble(statement, 15),
231		- sourceName: columnText(statement, 16),
	272	+ healthKitUUIDHash: columnText(statement, 4),
	273	+ startDate: columnUnixDate(statement, 5) ?? Date(timeIntervalSince1970: 0),
	274	+ endDate: columnUnixDate(statement, 6) ?? Date(timeIntervalSince1970: 0),
	275	+ firstSeenAt: columnUnixDate(statement, 7) ?? Date(timeIntervalSince1970: 0),
	276	+ lastSeenAt: columnUnixDate(statement, 8),
	277	+ lastVerifiedAt: columnUnixDate(statement, 9),
	278	+ disappearedAt: columnUnixDate(statement, 10),
	279	+ valueKind: columnText(statement, 11),
	280	+ value: columnDouble(statement, 12),
	281	+ unit: columnText(statement, 13),
	282	+ categoryValue: columnInt(statement, 14),
	283	+ workoutActivityType: columnInt(statement, 15),
	284	+ durationSeconds: columnDouble(statement, 16),
	285	+ sourceName: nil,
232	286	sourceBundleIdentifier: columnText(statement, 17),
233		- deviceName: columnText(statement, 18)
	287	+ deviceName: nil
234	288	))
235	289	}
236	290	return records
	@@ -997,13 +1051,14 @@ actor SQLiteHealthArchiveStore: HealthArchiveStore {
997	1051	AND (bundle_identifier = ? OR (bundle_identifier IS NULL AND ? IS NULL))
998	1052	LIMIT 1
999	1053	""",
1000		- db: db
1001		- ) { statement in
1002		- bindText(sourceNameHash, to: 1, in: statement)
1003		- bindText(sourceNameHash, to: 2, in: statement)
1004		- bindText(bundleIdentifier, to: 3, in: statement)
1005		- bindText(bundleIdentifier, to: 4, in: statement)
1006		- } {
	1054	+ db: db,
	1055	+ bind: { statement in
	1056	+ bindText(sourceNameHash, to: 1, in: statement)
	1057	+ bindText(sourceNameHash, to: 2, in: statement)
	1058	+ bindText(bundleIdentifier, to: 3, in: statement)
	1059	+ bindText(bundleIdentifier, to: 4, in: statement)
	1060	+ }
	1061	+ ) {
1007	1062	return existing
1008	1063	}
1009	1064	try withStatement("INSERT INTO sources (source_name_hash, bundle_identifier) VALUES (?, ?)", db: db) { statement in
	@@ -1032,17 +1087,18 @@ actor SQLiteHealthArchiveStore: HealthArchiveStore {
1032	1087	AND (model = ? OR (model IS NULL AND ? IS NULL))
1033	1088	LIMIT 1
1034	1089	""",
1035		- db: db
1036		- ) { statement in
1037		- bindText(deviceHash, to: 1, in: statement)
1038		- bindText(deviceHash, to: 2, in: statement)
1039		- bindText(localIdentifierHash, to: 3, in: statement)
1040		- bindText(localIdentifierHash, to: 4, in: statement)
1041		- bindText(udiHash, to: 5, in: statement)
1042		- bindText(udiHash, to: 6, in: statement)
1043		- bindText(row.deviceModel, to: 7, in: statement)
1044		- bindText(row.deviceModel, to: 8, in: statement)
1045		- } {
	1090	+ db: db,
	1091	+ bind: { statement in
	1092	+ bindText(deviceHash, to: 1, in: statement)
	1093	+ bindText(deviceHash, to: 2, in: statement)
	1094	+ bindText(localIdentifierHash, to: 3, in: statement)
	1095	+ bindText(localIdentifierHash, to: 4, in: statement)
	1096	+ bindText(udiHash, to: 5, in: statement)
	1097	+ bindText(udiHash, to: 6, in: statement)
	1098	+ bindText(row.deviceModel, to: 7, in: statement)
	1099	+ bindText(row.deviceModel, to: 8, in: statement)
	1100	+ }
	1101	+ ) {
1046	1102	return existing
1047	1103	}
1048	1104
	@@ -1818,7 +1874,7 @@ private struct ArchiveV2DailyAggregateRow {
1818	1874	let valueMax: Double?
1819	1875	let sourceRevisionID: Int64?
1820	1876
1821		- func hashParts(observationID: Int64, sampleTypeID: Int64) -> [String?] {
	1877	+ nonisolated func hashParts(observationID: Int64, sampleTypeID: Int64) -> [String?] {
1822	1878	[
1823	1879	String(observationID),
1824	1880	String(sampleTypeID),
	@@ -1893,6 +1949,11 @@ nonisolated private func columnDate(_ statement: OpaquePointer?, _ index: Int32)
1893	1949	return Date(timeIntervalSinceReferenceDate: sqlite3_column_double(statement, index))
1894	1950	}
1895	1951
	1952	+nonisolated private func columnUnixDate(_ statement: OpaquePointer?, _ index: Int32) -> Date? {
	1953	+ guard sqlite3_column_type(statement, index) != SQLITE_NULL else { return nil }
	1954	+ return Date(timeIntervalSince1970: sqlite3_column_double(statement, index))
	1955	+}
	1956	+
1896	1957	nonisolated private func columnDouble(_ statement: OpaquePointer?, _ index: Int32) -> Double? {
1897	1958	guard sqlite3_column_type(statement, index) != SQLITE_NULL else { return nil }
1898	1959	return sqlite3_column_double(statement, index)