- Defer grouped archive rebuilds until verification
- Browse files
-
bogdan committed a week ago
Showing 4 changed files
with
99 additions
and
24 deletions
+1
-1
HealthProbe/Doc/04-project/IMPLEMENTATION_STATUS.md
@@ -24,7 +24,7 @@ There are no real deployments, only test installations. Existing prototype datab |
||
| 24 | 24 |
| Area | Current Status | Target / Next Work | |
| 25 | 25 |
|------|----------------|--------------------| |
| 26 | 26 |
| Product docs | Updated | Keep `HealthProbe/Doc/README.md` as canonical index | |
| 27 |
-| HealthKit capture | Capture now opens one archive observation per user-visible snapshot, attaches HealthKit pages, deleted-object evidence, and type verification to that observation id before finishing it, and no longer aborts initial full-history imports after a fixed 30-minute wall-clock cap while page-level HealthKit timeouts remain in place | Continue moving UI/cache reads to archive-backed observation ids and revisit adaptive paging/background collection separately | |
|
| 27 |
+| HealthKit capture | Capture now opens one archive observation per user-visible snapshot, attaches HealthKit pages, deleted-object evidence, and type verification to that observation id before finishing it, no longer aborts initial full-history imports after a fixed 30-minute wall-clock cap while page-level HealthKit timeouts remain in place, and defers grouped observation summary/daily aggregate rebuilds until per-type verification instead of rebuilding after every imported page | Continue moving UI/cache reads to archive-backed observation ids and revisit adaptive paging/background collection separately | |
|
| 28 | 28 |
| SQLite archive | Archive v2 schema, snapshot-level observation grouping, differential write path, v2 verification/delete bookkeeping, daily aggregate rebuilds, integrity report, v2 record reads, SQL diff/count/aggregate/provenance/consolidation-evidence APIs, large synthetic diff pagination coverage, formal timing/memory metrics, and XCTest coverage are in place; the legacy `archive_samples` mirror has been removed | Continue moving capture/Dashboard actions to archive/cache DTOs | |
| 29 | 29 |
| Core Data cache | Initial programmatic Core Data model, full-cache rebuild service, read DTOs for observation/type/diff/health rows, and Dashboard archive-cache status wiring are in place | Move remaining export/report paths to cache DTOs and add targeted partial invalidation | |
| 30 | 30 |
| SwiftData cache | Exists; test builds now reset legacy prototype UI/archive/cache stores once for archive v2 so old SwiftData-only snapshots are not treated as backed-up observations. Metric timeout calibration, local device profile settings, operation logging, ContentView preview, Settings data maintenance, legacy detail/PDF views, unused legacy repair/observer services, Dashboard view/view-model access, and legacy anomaly/count-drop review have moved outside SwiftData or been removed. Remaining SwiftData imports are inventoried in [`SwiftData-Retirement-Inventory.md`](SwiftData-Retirement-Inventory.md) | Treat as disposable prototype data; stop returning/storing `HealthSnapshot` bridge handles before removing `ModelContainer` | |
+30
-0
HealthProbe/Services/HealthKitService.swift
@@ -1099,6 +1099,16 @@ final class HealthKitService {
|
||
| 1099 | 1099 |
anchor: anchor |
| 1100 | 1100 |
) |
| 1101 | 1101 |
} |
| 1102 |
+ progress?.updateBlockProgress( |
|
| 1103 |
+ typeIdentifier, |
|
| 1104 |
+ detail: "Persisting delta page \(pageNumber)", |
|
| 1105 |
+ recordCount: previousDistribution.count, |
|
| 1106 |
+ elapsedSeconds: Date().timeIntervalSince(progressStarted), |
|
| 1107 |
+ samplesPerSecond: Self.samplesPerSecond( |
|
| 1108 |
+ processedCount: processedEventCount, |
|
| 1109 |
+ elapsedSeconds: Date().timeIntervalSince(progressStarted) |
|
| 1110 |
+ ) |
|
| 1111 |
+ ) |
|
| 1102 | 1112 |
try await archivePage(page, sampleType: sampleType, observationID: archiveObservationID) |
| 1103 | 1113 |
anchor = page.anchor |
| 1104 | 1114 |
|
@@ -1186,6 +1196,16 @@ final class HealthKitService {
|
||
| 1186 | 1196 |
anchor: anchor |
| 1187 | 1197 |
) |
| 1188 | 1198 |
} |
| 1199 |
+ progress?.updateBlockProgress( |
|
| 1200 |
+ typeIdentifier, |
|
| 1201 |
+ detail: "Persisting delta page \(pageNumber)", |
|
| 1202 |
+ recordCount: recordMap.count, |
|
| 1203 |
+ elapsedSeconds: Date().timeIntervalSince(progressStarted), |
|
| 1204 |
+ samplesPerSecond: Self.samplesPerSecond( |
|
| 1205 |
+ processedCount: processedEventCount, |
|
| 1206 |
+ elapsedSeconds: Date().timeIntervalSince(progressStarted) |
|
| 1207 |
+ ) |
|
| 1208 |
+ ) |
|
| 1189 | 1209 |
try await archivePage(page, sampleType: sampleType, observationID: archiveObservationID) |
| 1190 | 1210 |
anchor = page.anchor |
| 1191 | 1211 |
|
@@ -1324,6 +1344,16 @@ final class HealthKitService {
|
||
| 1324 | 1344 |
anchor: anchor |
| 1325 | 1345 |
) |
| 1326 | 1346 |
} |
| 1347 |
+ progress?.updateBlockProgress( |
|
| 1348 |
+ typeIdentifier, |
|
| 1349 |
+ detail: "Persisting import page \(pageNumber)", |
|
| 1350 |
+ recordCount: recordCount, |
|
| 1351 |
+ elapsedSeconds: Date().timeIntervalSince(progressStarted), |
|
| 1352 |
+ samplesPerSecond: Self.samplesPerSecond( |
|
| 1353 |
+ processedCount: processedEventCount, |
|
| 1354 |
+ elapsedSeconds: Date().timeIntervalSince(progressStarted) |
|
| 1355 |
+ ) |
|
| 1356 |
+ ) |
|
| 1327 | 1357 |
try await archivePage(page, sampleType: sampleType, observationID: archiveObservationID) |
| 1328 | 1358 |
anchor = page.anchor |
| 1329 | 1359 |
|
+52
-23
HealthProbe/Services/SQLiteHealthArchiveStore.swift
@@ -95,7 +95,12 @@ actor SQLiteHealthArchiveStore: HealthArchiveStore {
|
||
| 95 | 95 |
try prepareSchemaIfNeeded(db) |
| 96 | 96 |
try execute("BEGIN IMMEDIATE TRANSACTION", db: db)
|
| 97 | 97 |
do {
|
| 98 |
- let summary = try upsertSamples(samples, observedAt: observedAt, db: db) |
|
| 98 |
+ let summary = try upsertSamples( |
|
| 99 |
+ samples, |
|
| 100 |
+ observedAt: observedAt, |
|
| 101 |
+ db: db, |
|
| 102 |
+ rebuildDerivedState: true |
|
| 103 |
+ ) |
|
| 99 | 104 |
try execute("PRAGMA foreign_key_check", db: db)
|
| 100 | 105 |
try execute("COMMIT", db: db)
|
| 101 | 106 |
return summary |
@@ -115,8 +120,13 @@ actor SQLiteHealthArchiveStore: HealthArchiveStore {
|
||
| 115 | 120 |
try prepareSchemaIfNeeded(db) |
| 116 | 121 |
try execute("BEGIN IMMEDIATE TRANSACTION", db: db)
|
| 117 | 122 |
do {
|
| 118 |
- let summary = try upsertSamples(samples, observedAt: observedAt, observationID: observationID, db: db) |
|
| 119 |
- try execute("PRAGMA foreign_key_check", db: db)
|
|
| 123 |
+ let summary = try upsertSamples( |
|
| 124 |
+ samples, |
|
| 125 |
+ observedAt: observedAt, |
|
| 126 |
+ observationID: observationID, |
|
| 127 |
+ db: db, |
|
| 128 |
+ rebuildDerivedState: false |
|
| 129 |
+ ) |
|
| 120 | 130 |
try execute("COMMIT", db: db)
|
| 121 | 131 |
return summary |
| 122 | 132 |
} catch {
|
@@ -176,7 +186,8 @@ actor SQLiteHealthArchiveStore: HealthArchiveStore {
|
||
| 176 | 186 |
sampleTypeIdentifier: sampleTypeIdentifier, |
| 177 | 187 |
observedMissingAt: observedMissingAt, |
| 178 | 188 |
observationID: observationID, |
| 179 |
- db: db |
|
| 189 |
+ db: db, |
|
| 190 |
+ rebuildDerivedState: true |
|
| 180 | 191 |
) |
| 181 | 192 |
|
| 182 | 193 |
try execute("COMMIT", db: db)
|
@@ -202,7 +213,8 @@ actor SQLiteHealthArchiveStore: HealthArchiveStore {
|
||
| 202 | 213 |
sampleTypeIdentifier: sampleTypeIdentifier, |
| 203 | 214 |
observedMissingAt: observedMissingAt, |
| 204 | 215 |
observationID: observationID, |
| 205 |
- db: db |
|
| 216 |
+ db: db, |
|
| 217 |
+ rebuildDerivedState: false |
|
| 206 | 218 |
) |
| 207 | 219 |
try execute("COMMIT", db: db)
|
| 208 | 220 |
} catch {
|
@@ -1768,21 +1780,33 @@ actor SQLiteHealthArchiveStore: HealthArchiveStore {
|
||
| 1768 | 1780 |
} |
| 1769 | 1781 |
} |
| 1770 | 1782 |
|
| 1771 |
- private func upsertSamples(_ samples: [HKSample], observedAt: Date, db: OpaquePointer?) throws -> HealthArchiveWriteSummary {
|
|
| 1783 |
+ private func upsertSamples( |
|
| 1784 |
+ _ samples: [HKSample], |
|
| 1785 |
+ observedAt: Date, |
|
| 1786 |
+ db: OpaquePointer?, |
|
| 1787 |
+ rebuildDerivedState: Bool |
|
| 1788 |
+ ) throws -> HealthArchiveWriteSummary {
|
|
| 1772 | 1789 |
let observationID = try createObservation( |
| 1773 | 1790 |
observedAt: observedAt, |
| 1774 | 1791 |
triggerReason: "anchored_page", |
| 1775 | 1792 |
status: "completed", |
| 1776 | 1793 |
db: db |
| 1777 | 1794 |
) |
| 1778 |
- return try upsertSamples(samples, observedAt: observedAt, observationID: observationID, db: db) |
|
| 1795 |
+ return try upsertSamples( |
|
| 1796 |
+ samples, |
|
| 1797 |
+ observedAt: observedAt, |
|
| 1798 |
+ observationID: observationID, |
|
| 1799 |
+ db: db, |
|
| 1800 |
+ rebuildDerivedState: rebuildDerivedState |
|
| 1801 |
+ ) |
|
| 1779 | 1802 |
} |
| 1780 | 1803 |
|
| 1781 | 1804 |
private func upsertSamples( |
| 1782 | 1805 |
_ samples: [HKSample], |
| 1783 | 1806 |
observedAt: Date, |
| 1784 | 1807 |
observationID: Int64, |
| 1785 |
- db: OpaquePointer? |
|
| 1808 |
+ db: OpaquePointer?, |
|
| 1809 |
+ rebuildDerivedState: Bool |
|
| 1786 | 1810 |
) throws -> HealthArchiveWriteSummary {
|
| 1787 | 1811 |
let rows = samples.map { ArchiveSampleRow(sample: $0, observedAt: observedAt) }
|
| 1788 | 1812 |
var inserted = 0 |
@@ -1817,13 +1841,15 @@ actor SQLiteHealthArchiveStore: HealthArchiveStore {
|
||
| 1817 | 1841 |
verifiedVisibleCount: nil, |
| 1818 | 1842 |
db: db |
| 1819 | 1843 |
) |
| 1820 |
- try rebuildTypeSummary(observationID: observationID, sampleTypeID: sampleTypeID, db: db) |
|
| 1821 |
- try rebuildDailyAggregates( |
|
| 1822 |
- observationID: observationID, |
|
| 1823 |
- sampleTypeID: sampleTypeID, |
|
| 1824 |
- observedAt: observedAt, |
|
| 1825 |
- db: db |
|
| 1826 |
- ) |
|
| 1844 |
+ if rebuildDerivedState {
|
|
| 1845 |
+ try rebuildTypeSummary(observationID: observationID, sampleTypeID: sampleTypeID, db: db) |
|
| 1846 |
+ try rebuildDailyAggregates( |
|
| 1847 |
+ observationID: observationID, |
|
| 1848 |
+ sampleTypeID: sampleTypeID, |
|
| 1849 |
+ observedAt: observedAt, |
|
| 1850 |
+ db: db |
|
| 1851 |
+ ) |
|
| 1852 |
+ } |
|
| 1827 | 1853 |
} |
| 1828 | 1854 |
|
| 1829 | 1855 |
return HealthArchiveWriteSummary( |
@@ -1865,7 +1891,8 @@ actor SQLiteHealthArchiveStore: HealthArchiveStore {
|
||
| 1865 | 1891 |
sampleTypeIdentifier: String, |
| 1866 | 1892 |
observedMissingAt: Date, |
| 1867 | 1893 |
observationID: Int64, |
| 1868 |
- db: OpaquePointer? |
|
| 1894 |
+ db: OpaquePointer?, |
|
| 1895 |
+ rebuildDerivedState: Bool |
|
| 1869 | 1896 |
) throws {
|
| 1870 | 1897 |
guard let sampleTypeID = try sampleTypeID(typeIdentifier: sampleTypeIdentifier, db: db), |
| 1871 | 1898 |
let sampleID = try sampleID(sampleUUIDHash: sampleUUIDHash, sampleTypeID: sampleTypeID, db: db) else {
|
@@ -1898,13 +1925,15 @@ actor SQLiteHealthArchiveStore: HealthArchiveStore {
|
||
| 1898 | 1925 |
verifiedVisibleCount: nil, |
| 1899 | 1926 |
db: db |
| 1900 | 1927 |
) |
| 1901 |
- try rebuildTypeSummary(observationID: observationID, sampleTypeID: sampleTypeID, db: db) |
|
| 1902 |
- try rebuildDailyAggregates( |
|
| 1903 |
- observationID: observationID, |
|
| 1904 |
- sampleTypeID: sampleTypeID, |
|
| 1905 |
- observedAt: observedMissingAt, |
|
| 1906 |
- db: db |
|
| 1907 |
- ) |
|
| 1928 |
+ if rebuildDerivedState {
|
|
| 1929 |
+ try rebuildTypeSummary(observationID: observationID, sampleTypeID: sampleTypeID, db: db) |
|
| 1930 |
+ try rebuildDailyAggregates( |
|
| 1931 |
+ observationID: observationID, |
|
| 1932 |
+ sampleTypeID: sampleTypeID, |
|
| 1933 |
+ observedAt: observedMissingAt, |
|
| 1934 |
+ db: db |
|
| 1935 |
+ ) |
|
| 1936 |
+ } |
|
| 1908 | 1937 |
} |
| 1909 | 1938 |
|
| 1910 | 1939 |
private func upsertArchiveV2Sample( |
+16
-0
HealthProbeTests/SQLiteHealthArchiveStoreTests.swift
@@ -219,6 +219,20 @@ final class SQLiteHealthArchiveStoreTests: XCTestCase {
|
||
| 219 | 219 |
observedMissingAt: Date(timeIntervalSince1970: 3_060), |
| 220 | 220 |
observationID: observationID |
| 221 | 221 |
) |
| 222 |
+ XCTAssertEqual( |
|
| 223 |
+ try countRows( |
|
| 224 |
+ in: "observation_type_summaries WHERE observation_id = \(observationID)", |
|
| 225 |
+ at: url |
|
| 226 |
+ ), |
|
| 227 |
+ 0 |
|
| 228 |
+ ) |
|
| 229 |
+ XCTAssertEqual( |
|
| 230 |
+ try countRows( |
|
| 231 |
+ in: "daily_type_aggregates WHERE observation_id = \(observationID)", |
|
| 232 |
+ at: url |
|
| 233 |
+ ), |
|
| 234 |
+ 0 |
|
| 235 |
+ ) |
|
| 222 | 236 |
try await store.markVerification( |
| 223 | 237 |
sampleType: secondSample.sampleType, |
| 224 | 238 |
verifiedAt: Date(timeIntervalSince1970: 3_060), |
@@ -243,6 +257,8 @@ final class SQLiteHealthArchiveStoreTests: XCTestCase {
|
||
| 243 | 257 |
XCTAssertEqual(try countRows(in: "observations WHERE id = \(observationID) AND status = 'completed' AND selected_type_set_hash = 'selected-types'", at: url), 1) |
| 244 | 258 |
XCTAssertEqual(try countRows(in: "sample_observation_events WHERE observation_id = \(observationID)", at: url), 2) |
| 245 | 259 |
XCTAssertEqual(try countRows(in: "observation_type_runs WHERE observation_id = \(observationID) AND inserted_event_count = 1 AND deleted_event_count = 1 AND verified_visible_count = 1 AND status = 'completed'", at: url), 1) |
| 260 |
+ XCTAssertEqual(try countRows(in: "observation_type_summaries WHERE observation_id = \(observationID) AND visible_record_count = 1", at: url), 1) |
|
| 261 |
+ XCTAssertGreaterThan(try countRows(in: "daily_type_aggregates WHERE observation_id = \(observationID)", at: url), 0) |
|
| 246 | 262 |
} |
| 247 | 263 |
|
| 248 | 264 |
func testLargeSyntheticDiffReturnsCountsAndBoundedPages() async throws {
|