HealthProbe / HealthProbeTests / SQLiteHealthArchiveStoreTests.swift
Newer Older
735 lines | 35.151kb
Bogdan Timofte authored 2 weeks ago
1
import HealthKit
2
import SQLite3
3
import XCTest
4
@testable import HealthProbe
5

            
6
final class SQLiteHealthArchiveStoreTests: XCTestCase {
Bogdan Timofte authored 2 weeks ago
7
    private final class AsyncResultBox<T>: @unchecked Sendable {
8
        var result: Result<T, Error>?
9
    }
10

            
Bogdan Timofte authored 2 weeks ago
11
    private var temporaryDirectory: URL!
12

            
13
    override func setUpWithError() throws {
14
        temporaryDirectory = FileManager.default.temporaryDirectory
15
            .appending(path: "HealthProbeTests-\(UUID().uuidString)", directoryHint: .isDirectory)
16
        try FileManager.default.createDirectory(at: temporaryDirectory, withIntermediateDirectories: true)
17
    }
18

            
19
    override func tearDownWithError() throws {
20
        if let temporaryDirectory {
21
            try? FileManager.default.removeItem(at: temporaryDirectory)
22
        }
23
        temporaryDirectory = nil
24
    }
25

            
26
    func testFreshArchiveInitializesSchemaAndPassesIntegrity() async throws {
27
        let store = SQLiteHealthArchiveStore(databaseURL: databaseURL())
28

            
29
        let report = try await store.checkIntegrity()
30
        let records = try await store.records(for: HealthArchiveRecordRequest(
31
            sampleTypeIdentifier: HKQuantityTypeIdentifier.stepCount.rawValue
32
        ))
33

            
34
        XCTAssertTrue(report.passed)
35
        XCTAssertEqual(report.schemaVersion, 2)
36
        XCTAssertEqual(report.sqliteIntegrityStatus, "ok")
37
        XCTAssertEqual(report.foreignKeyIssueCount, 0)
38
        XCTAssertTrue(report.missingTableNames.isEmpty)
39
        XCTAssertTrue(report.requiredTableNames.contains("sample_visibility_ranges"))
40
        XCTAssertTrue(report.requiredTableNames.contains("daily_type_aggregates"))
41
        XCTAssertTrue(records.isEmpty)
42
    }
43

            
44
    func testPrototypeArchiveIsResetAndReinitializedAsV2() async throws {
45
        let url = databaseURL()
46
        try createPrototypeDatabase(at: url)
47
        let store = SQLiteHealthArchiveStore(databaseURL: url)
48

            
49
        let report = try await store.checkIntegrity()
50

            
51
        XCTAssertTrue(report.passed)
52
        XCTAssertEqual(report.schemaVersion, 2)
53
        XCTAssertTrue(report.missingTableNames.isEmpty)
54
    }
55

            
56
    func testRepeatedSamplePageDoesNotDuplicateIdentityOrVersion() async throws {
57
        let url = databaseURL()
58
        let store = SQLiteHealthArchiveStore(databaseURL: url)
59
        let sample = makeStepCountSample()
60

            
61
        let firstWrite = try await store.upsertSamples([sample], observedAt: Date(timeIntervalSince1970: 2_000))
62
        let secondWrite = try await store.upsertSamples([sample], observedAt: Date(timeIntervalSince1970: 2_060))
63
        let records = try await store.records(for: HealthArchiveRecordRequest(
64
            sampleTypeIdentifier: sample.sampleType.identifier
65
        ))
66
        let report = try await store.checkIntegrity()
67
        let versionDebugRows = try sampleVersionDebugRows(at: url)
Bogdan Timofte authored a week ago
68
        let visibilityDebugRows = try visibilityRangeDebugRows(at: url)
Bogdan Timofte authored 2 weeks ago
69

            
70
        XCTAssertEqual(firstWrite.insertedCount, 1)
71
        XCTAssertEqual(firstWrite.updatedCount, 0)
72
        XCTAssertEqual(firstWrite.unchangedCount, 0)
73
        XCTAssertEqual(try countRows(in: "samples", at: url), 1)
74
        XCTAssertEqual(try countRows(in: "sample_versions", at: url), 1, versionDebugRows)
Bogdan Timofte authored 5 days ago
75
        XCTAssertEqual(try countRows(in: "sample_observation_events", at: url), 1)
76
        XCTAssertEqual(try countRows(in: "sample_observation_events WHERE event_kind = 'appeared'", at: url), 1)
77
        XCTAssertEqual(try countRows(in: "sample_observation_events WHERE event_kind = 'verified'", at: url), 0)
Bogdan Timofte authored a week ago
78
        XCTAssertEqual(try countRows(in: "sample_visibility_ranges", at: url), 1, visibilityDebugRows)
Bogdan Timofte authored 2 weeks ago
79
        XCTAssertEqual(try countRows(in: "source_revisions", at: url), 1)
Bogdan Timofte authored 2 weeks ago
80
        XCTAssertFalse(try tableExists("archive_samples", at: url))
Bogdan Timofte authored 2 weeks ago
81
        XCTAssertEqual(secondWrite.insertedCount, 0)
82
        XCTAssertEqual(secondWrite.updatedCount, 0)
83
        XCTAssertEqual(secondWrite.unchangedCount, 1)
84
        XCTAssertEqual(records.count, 1)
85
        XCTAssertEqual(records.first?.displayValue, "42.0 count")
86
        XCTAssertTrue(report.passed)
87
    }
88

            
Bogdan Timofte authored 2 weeks ago
89
    func testVerificationUsesArchiveV2TablesWithoutLegacyMirror() async throws {
90
        let url = databaseURL()
91
        let store = SQLiteHealthArchiveStore(databaseURL: url)
92
        let sample = makeStepCountSample()
93

            
94
        _ = try await store.upsertSamples([sample], observedAt: Date(timeIntervalSince1970: 2_000))
95
        try await store.markVerification(sampleType: sample.sampleType, verifiedAt: Date(timeIntervalSince1970: 2_060))
96
        let observationIDs = try observationIDs(at: url)
97

            
98
        XCTAssertEqual(observationIDs.count, 2)
Bogdan Timofte authored 2 weeks ago
99
        XCTAssertEqual(try countRows(in: "observation_type_runs", at: url), 2)
100
        XCTAssertEqual(try countRows(in: "observation_type_runs WHERE observation_id = \(observationIDs[0]) AND inserted_event_count = 1", at: url), 1)
Bogdan Timofte authored 2 weeks ago
101
        XCTAssertEqual(try countRows(in: "observation_type_summaries WHERE observation_id = \(observationIDs[1]) AND visible_record_count = 1", at: url), 1)
Bogdan Timofte authored 5 days ago
102
        XCTAssertEqual(try countRows(in: "daily_type_aggregates WHERE observation_id = \(observationIDs[1]) AND visible_record_count = 1", at: url), 1)
Bogdan Timofte authored 2 weeks ago
103
        XCTAssertFalse(try tableExists("archive_samples", at: url))
104
    }
105

            
Bogdan Timofte authored 2 weeks ago
106
    func testDiffSummaryAndRecordsBetweenObservationsUseSQLVisibility() async throws {
107
        let url = databaseURL()
108
        let store = SQLiteHealthArchiveStore(databaseURL: url)
109
        let firstSample = makeStepCountSample(value: 42, start: 1_000)
110
        let secondSample = makeStepCountSample(value: 7, start: 2_000)
111
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
112

            
113
        _ = try await store.upsertSamples([firstSample], observedAt: Date(timeIntervalSince1970: 3_000))
114
        _ = try await store.upsertSamples([firstSample, secondSample], observedAt: Date(timeIntervalSince1970: 3_060))
115
        try await store.recordDisappearance(
116
            sampleUUIDHash: HashService.sampleUUIDHash(firstSample.uuid.uuidString),
117
            sampleTypeIdentifier: typeIdentifier,
118
            observedMissingAt: Date(timeIntervalSince1970: 3_120)
119
        )
120
        let observationIDs = try observationIDs(at: url)
121
        XCTAssertEqual(observationIDs.count, 3)
122

            
123
        let appearedSummary = try await store.diffSummary(HealthArchiveDiffRequest(
124
            fromObservationID: observationIDs[0],
125
            toObservationID: observationIDs[1],
126
            sampleTypeIdentifier: typeIdentifier
127
        ))
128
        let appearedRecords = try await store.diffRecords(HealthArchiveDiffRecordRequest(
129
            fromObservationID: observationIDs[0],
130
            toObservationID: observationIDs[1],
131
            sampleTypeIdentifier: typeIdentifier,
132
            kind: .appeared,
133
            limit: 10
134
        ))
135
        let disappearedSummary = try await store.diffSummary(HealthArchiveDiffRequest(
136
            fromObservationID: observationIDs[1],
137
            toObservationID: observationIDs[2],
138
            sampleTypeIdentifier: typeIdentifier
139
        ))
140
        let disappearedRecords = try await store.diffRecords(HealthArchiveDiffRecordRequest(
141
            fromObservationID: observationIDs[1],
142
            toObservationID: observationIDs[2],
143
            sampleTypeIdentifier: typeIdentifier,
144
            kind: .disappeared,
145
            limit: 10
146
        ))
147

            
148
        XCTAssertEqual(appearedSummary.appearedCount, 1)
149
        XCTAssertEqual(appearedSummary.disappearedCount, 0)
150
        XCTAssertEqual(appearedSummary.representationChangedCount, 0)
151
        XCTAssertEqual(appearedRecords.map(\.displayValue), ["7.0 count"])
152
        XCTAssertEqual(disappearedSummary.appearedCount, 0)
153
        XCTAssertEqual(disappearedSummary.disappearedCount, 1)
154
        XCTAssertEqual(disappearedSummary.representationChangedCount, 0)
155
        XCTAssertEqual(disappearedRecords.map(\.displayValue), ["42.0 count"])
156
        XCTAssertNotNil(disappearedRecords.first?.disappearedAt)
Bogdan Timofte authored 2 weeks ago
157

            
158
        let exportRequest = HealthArchiveReportRequest(
159
            reportID: UUID(),
160
            title: "Disappeared Step Count Export",
161
            typeIdentifierFilter: typeIdentifier,
162
            diffFromObservationID: observationIDs[1],
163
            diffToObservationID: observationIDs[2],
164
            diffKind: .disappeared
165
        )
166
        let exportPreview = try await store.exportPreview(exportRequest)
167
        let exportURL = try await store.exportReport(exportRequest)
168

            
169
        XCTAssertEqual(exportPreview.estimatedRecordCount, 1)
170
        XCTAssertTrue(FileManager.default.fileExists(atPath: exportURL.path))
171
        XCTAssertEqual(try countRows(in: "export_manifests WHERE record_count = 1 AND from_observation_id = \(observationIDs[1]) AND to_observation_id = \(observationIDs[2])", at: url), 1)
172
    }
173

            
174
    func testExportPreviewAndReportUseSQLiteManifest() async throws {
175
        let url = databaseURL()
176
        let store = SQLiteHealthArchiveStore(databaseURL: url)
177
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
178
        let samples = [
179
            makeStepCountSample(value: 42, start: 1_000),
180
            makeStepCountSample(value: 7, start: 2_000)
181
        ]
182

            
183
        _ = try await store.upsertSamples(samples, observedAt: Date(timeIntervalSince1970: 3_000))
184

            
185
        let request = HealthArchiveReportRequest(
186
            reportID: UUID(),
187
            title: "Step Count Export",
188
            typeIdentifierFilter: typeIdentifier
189
        )
190
        let preview = try await store.exportPreview(request)
191
        let exportURL = try await store.exportReport(request)
192

            
193
        XCTAssertEqual(preview.estimatedRecordCount, 2)
194
        XCTAssertTrue(FileManager.default.fileExists(atPath: exportURL.path))
195
        XCTAssertEqual(try countRows(in: "export_manifests", at: url), 1)
196
        XCTAssertEqual(try countRows(in: "export_manifests WHERE record_count = 2", at: url), 1)
197

            
198
        let cache = try CoreDataArchiveCacheStore(inMemory: true)
199
        let rebuild = try cache.rebuild(fromArchiveAt: url)
200
        XCTAssertEqual(rebuild.exportManifestRows, 1)
Bogdan Timofte authored 2 weeks ago
201
    }
202

            
Bogdan Timofte authored 2 weeks ago
203
    func testGroupedObservationKeepsPageWritesDeletesAndVerificationTogether() async throws {
204
        let url = databaseURL()
205
        let store = SQLiteHealthArchiveStore(databaseURL: url)
206
        let firstSample = makeStepCountSample(value: 42, start: 1_000)
207
        let secondSample = makeStepCountSample(value: 7, start: 2_000)
208
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
209

            
210
        _ = try await store.upsertSamples([firstSample], observedAt: Date(timeIntervalSince1970: 3_000))
211
        let observationID = try await store.beginObservation(
212
            observedAt: Date(timeIntervalSince1970: 3_060),
213
            triggerReason: "manual",
214
            selectedTypeSetHash: "selected-types"
215
        )
216
        _ = try await store.upsertSamples(
217
            [secondSample],
218
            observedAt: Date(timeIntervalSince1970: 3_060),
219
            observationID: observationID
220
        )
221
        try await store.recordDisappearance(
222
            sampleUUIDHash: HashService.sampleUUIDHash(firstSample.uuid.uuidString),
223
            sampleTypeIdentifier: typeIdentifier,
224
            observedMissingAt: Date(timeIntervalSince1970: 3_060),
225
            observationID: observationID
226
        )
Bogdan Timofte authored a week ago
227
        XCTAssertEqual(
228
            try countRows(
229
                in: "observation_type_summaries WHERE observation_id = \(observationID)",
230
                at: url
231
            ),
232
            0
233
        )
234
        XCTAssertEqual(
235
            try countRows(
236
                in: "daily_type_aggregates WHERE observation_id = \(observationID)",
237
                at: url
238
            ),
239
            0
240
        )
Bogdan Timofte authored 2 weeks ago
241
        try await store.markVerification(
242
            sampleType: secondSample.sampleType,
243
            verifiedAt: Date(timeIntervalSince1970: 3_060),
244
            observationID: observationID
245
        )
246
        try await store.finishObservation(
247
            observationID: observationID,
248
            status: "completed",
249
            endedAt: Date(timeIntervalSince1970: 3_070)
250
        )
251

            
252
        let observationIDs = try observationIDs(at: url)
253
        let summary = try await store.diffSummary(HealthArchiveDiffRequest(
254
            fromObservationID: observationIDs[0],
255
            toObservationID: observationID,
256
            sampleTypeIdentifier: typeIdentifier
257
        ))
258

            
259
        XCTAssertEqual(observationIDs.count, 2)
260
        XCTAssertEqual(summary.appearedCount, 1)
261
        XCTAssertEqual(summary.disappearedCount, 1)
262
        XCTAssertEqual(try countRows(in: "observations WHERE id = \(observationID) AND status = 'completed' AND selected_type_set_hash = 'selected-types'", at: url), 1)
263
        XCTAssertEqual(try countRows(in: "sample_observation_events WHERE observation_id = \(observationID)", at: url), 2)
264
        XCTAssertEqual(try countRows(in: "observation_type_runs WHERE observation_id = \(observationID) AND inserted_event_count = 1 AND deleted_event_count = 1 AND verified_visible_count = 1 AND status = 'completed'", at: url), 1)
Bogdan Timofte authored a week ago
265
        XCTAssertEqual(try countRows(in: "observation_type_summaries WHERE observation_id = \(observationID) AND visible_record_count = 1", at: url), 1)
266
        XCTAssertGreaterThan(try countRows(in: "daily_type_aggregates WHERE observation_id = \(observationID)", at: url), 0)
Bogdan Timofte authored 2 weeks ago
267
    }
268

            
Bogdan Timofte authored a week ago
269
    func testGroupedObservationCanBatchDeletedObjectsInSingleRun() async throws {
270
        let url = databaseURL()
271
        let store = SQLiteHealthArchiveStore(databaseURL: url)
272
        let samples = [
273
            makeStepCountSample(value: 42, start: 1_000),
274
            makeStepCountSample(value: 7, start: 2_000),
275
            makeStepCountSample(value: 9, start: 3_000)
276
        ]
277
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
278

            
279
        _ = try await store.upsertSamples(samples, observedAt: Date(timeIntervalSince1970: 4_000))
280
        let observationID = try await store.beginObservation(
281
            observedAt: Date(timeIntervalSince1970: 4_060),
282
            triggerReason: "manual",
283
            selectedTypeSetHash: "selected-types"
284
        )
285
        let deletedCount = try await store.recordDisappearances(
286
            sampleUUIDHashes: samples.map { HashService.sampleUUIDHash($0.uuid.uuidString) },
287
            sampleTypeIdentifier: typeIdentifier,
288
            observedMissingAt: Date(timeIntervalSince1970: 4_060),
289
            observationID: observationID
290
        )
291
        try await store.markVerification(
292
            sampleType: samples[0].sampleType,
293
            verifiedAt: Date(timeIntervalSince1970: 4_060),
294
            observationID: observationID
295
        )
296

            
297
        XCTAssertEqual(deletedCount, 3)
298
        XCTAssertEqual(try countRows(in: "sample_observation_events WHERE observation_id = \(observationID) AND event_kind = 'disappeared'", at: url), 3)
299
        XCTAssertEqual(try countRows(in: "observation_type_runs WHERE observation_id = \(observationID) AND deleted_event_count = 3 AND status = 'completed'", at: url), 1)
300
    }
301

            
Bogdan Timofte authored 2 weeks ago
302
    func testLargeSyntheticDiffReturnsCountsAndBoundedPages() async throws {
303
        let url = databaseURL()
304
        let store = SQLiteHealthArchiveStore(databaseURL: url)
305
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
306
        let initialCount = 1_200
307
        let appearedCount = 180
308
        let disappearedCount = 160
309
        let pageSize = 25
310
        let initialSamples = makeStepCountSamples(count: initialCount, startIndex: 0)
311
        let appearedSamples = makeStepCountSamples(count: appearedCount, startIndex: initialCount)
312

            
313
        _ = try await store.upsertSamples(initialSamples, observedAt: Date(timeIntervalSince1970: 1_000_000))
314
        _ = try await store.upsertSamples(initialSamples + appearedSamples, observedAt: Date(timeIntervalSince1970: 1_000_060))
315
        for (offset, sample) in initialSamples.prefix(disappearedCount).enumerated() {
316
            try await store.recordDisappearance(
317
                sampleUUIDHash: HashService.sampleUUIDHash(sample.uuid.uuidString),
318
                sampleTypeIdentifier: typeIdentifier,
319
                observedMissingAt: Date(timeIntervalSince1970: 1_000_120 + Double(offset))
320
            )
321
        }
322

            
323
        let observationIDs = try observationIDs(at: url)
324
        let firstObservationID = try XCTUnwrap(observationIDs.first)
325
        let lastObservationID = try XCTUnwrap(observationIDs.last)
326
        let queryStartedAt = Date()
327
        let summary = try await store.diffSummary(HealthArchiveDiffRequest(
328
            fromObservationID: firstObservationID,
329
            toObservationID: lastObservationID,
330
            sampleTypeIdentifier: typeIdentifier
331
        ))
332
        let firstPage = try await store.diffRecords(HealthArchiveDiffRecordRequest(
333
            fromObservationID: firstObservationID,
334
            toObservationID: lastObservationID,
335
            sampleTypeIdentifier: typeIdentifier,
336
            kind: .appeared,
337
            limit: pageSize
338
        ))
339
        let firstPageLastRecord = try XCTUnwrap(firstPage.last)
340
        let secondPage = try await store.diffRecords(HealthArchiveDiffRecordRequest(
341
            fromObservationID: firstObservationID,
342
            toObservationID: lastObservationID,
343
            sampleTypeIdentifier: typeIdentifier,
344
            kind: .appeared,
345
            afterCursor: RecordCursor(
346
                startDate: firstPageLastRecord.startDate,
347
                strictFingerprint: firstPageLastRecord.strictFingerprint
348
            ),
349
            limit: pageSize
350
        ))
351
        let queryElapsedSeconds = Date().timeIntervalSince(queryStartedAt)
352

            
353
        XCTAssertEqual(summary.appearedCount, appearedCount)
354
        XCTAssertEqual(summary.disappearedCount, disappearedCount)
355
        XCTAssertEqual(summary.representationChangedCount, 0)
356
        XCTAssertEqual(firstPage.count, pageSize)
357
        XCTAssertEqual(secondPage.count, pageSize)
358
        XCTAssertTrue(Set(firstPage.map(\.id)).isDisjoint(with: Set(secondPage.map(\.id))))
359
        XCTAssertLessThan(queryElapsedSeconds, 10)
360
    }
361

            
Bogdan Timofte authored 2 weeks ago
362
    func testLargeSyntheticDiffQueryMetrics() throws {
363
        let url = databaseURL()
364
        let store = SQLiteHealthArchiveStore(databaseURL: url)
365
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
366
        let initialCount = 900
367
        let appearedCount = 120
368
        let disappearedCount = 100
369
        let pageSize = 25
370
        let initialSamples = makeStepCountSamples(count: initialCount, startIndex: 0)
371
        let appearedSamples = makeStepCountSamples(count: appearedCount, startIndex: initialCount)
372

            
373
        try waitForArchiveOperation {
374
            _ = try await store.upsertSamples(initialSamples, observedAt: Date(timeIntervalSince1970: 1_200_000))
375
            _ = try await store.upsertSamples(initialSamples + appearedSamples, observedAt: Date(timeIntervalSince1970: 1_200_060))
376
            for (offset, sample) in initialSamples.prefix(disappearedCount).enumerated() {
377
                try await store.recordDisappearance(
378
                    sampleUUIDHash: HashService.sampleUUIDHash(sample.uuid.uuidString),
379
                    sampleTypeIdentifier: typeIdentifier,
380
                    observedMissingAt: Date(timeIntervalSince1970: 1_200_120 + Double(offset))
381
                )
382
            }
383
        }
384

            
385
        let observationIDs = try observationIDs(at: url)
386
        let firstObservationID = try XCTUnwrap(observationIDs.first)
387
        let lastObservationID = try XCTUnwrap(observationIDs.last)
388
        let options = XCTMeasureOptions()
389
        options.iterationCount = 3
390

            
391
        measure(metrics: [XCTClockMetric(), XCTMemoryMetric()], options: options) {
392
            do {
393
                let result = try waitForArchiveOperation {
394
                    let summary = try await store.diffSummary(HealthArchiveDiffRequest(
395
                        fromObservationID: firstObservationID,
396
                        toObservationID: lastObservationID,
397
                        sampleTypeIdentifier: typeIdentifier
398
                    ))
399
                    let records = try await store.diffRecords(HealthArchiveDiffRecordRequest(
400
                        fromObservationID: firstObservationID,
401
                        toObservationID: lastObservationID,
402
                        sampleTypeIdentifier: typeIdentifier,
403
                        kind: .appeared,
404
                        limit: pageSize
405
                    ))
406
                    return (summary, records)
407
                }
408
                XCTAssertEqual(result.0.appearedCount, appearedCount)
409
                XCTAssertEqual(result.0.disappearedCount, disappearedCount)
410
                XCTAssertEqual(result.1.count, pageSize)
411
            } catch {
412
                XCTFail("Measured archive query failed: \(error)")
413
            }
414
        }
415
    }
416

            
Bogdan Timofte authored 2 weeks ago
417
    func testAggregateComparisonUsesMaterializedDailyAggregates() async throws {
418
        let url = databaseURL()
419
        let store = SQLiteHealthArchiveStore(databaseURL: url)
420
        let firstSample = makeStepCountSample(value: 42, start: 1_000)
421
        let secondSample = makeStepCountSample(value: 7, start: 2_000)
422
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
423

            
424
        _ = try await store.upsertSamples([firstSample], observedAt: Date(timeIntervalSince1970: 3_000))
425
        _ = try await store.upsertSamples([firstSample, secondSample], observedAt: Date(timeIntervalSince1970: 3_060))
426
        let observationIDs = try observationIDs(at: url)
427
        XCTAssertEqual(observationIDs.count, 2)
428

            
429
        let rows = try await store.aggregateComparison(HealthArchiveAggregateComparisonRequest(
430
            fromObservationID: observationIDs[0],
431
            toObservationID: observationIDs[1],
432
            sampleTypeIdentifier: typeIdentifier,
433
            limit: 10
434
        ))
435

            
436
        XCTAssertEqual(rows.count, 1)
437
        XCTAssertEqual(rows.first?.sampleTypeIdentifier, typeIdentifier)
438
        XCTAssertEqual(rows.first?.fromVisibleRecordCount, 1)
439
        XCTAssertEqual(rows.first?.toVisibleRecordCount, 2)
440
        XCTAssertEqual(rows.first?.visibleRecordDelta, 1)
441
        XCTAssertEqual(rows.first?.fromValueSum, 42)
442
        XCTAssertEqual(rows.first?.toValueSum, 49)
443
        XCTAssertEqual(rows.first?.valueSumDelta, 7)
444
    }
445

            
Bogdan Timofte authored 2 weeks ago
446
    func testSourceProvenanceBreakdownReturnsVisibleSourceRows() async throws {
447
        let url = databaseURL()
448
        let store = SQLiteHealthArchiveStore(databaseURL: url)
449
        let sample = makeStepCountSample(value: 42, start: 1_000)
450
        let typeIdentifier = sample.sampleType.identifier
451

            
452
        _ = try await store.upsertSamples([sample], observedAt: Date(timeIntervalSince1970: 3_000))
453

            
454
        let rows = try await store.sourceProvenanceBreakdown(HealthArchiveSourceProvenanceRequest(
455
            visibleAtObservationID: nil,
456
            sampleTypeIdentifier: typeIdentifier,
457
            limit: 10
458
        ))
459

            
460
        XCTAssertEqual(rows.count, 1)
461
        XCTAssertEqual(rows.first?.sampleTypeIdentifier, typeIdentifier)
462
        XCTAssertEqual(rows.first?.visibleRecordCount, 1)
463
        XCTAssertEqual(rows.first?.valueSum, 42)
464
        XCTAssertEqual(rows.first?.sourceBundleIdentifier, sample.sourceRevision.source.bundleIdentifier)
465
    }
466

            
467
    func testConsolidationEvidenceClassifiesStableCountDropAsConsolidationLikely() async throws {
468
        let url = databaseURL()
469
        let store = SQLiteHealthArchiveStore(databaseURL: url)
470
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
471

            
472
        let firstOldSample = makeStepCountSample(value: 10, start: 1_000, end: 1_500)
473
        let secondOldSample = makeStepCountSample(value: 20, start: 1_600, end: 2_000)
474
        let consolidatedSample = makeStepCountSample(value: 30, start: 1_000, end: 2_500)
475

            
476
        _ = try await store.upsertSamples([firstOldSample, secondOldSample], observedAt: Date(timeIntervalSince1970: 3_000))
477
        _ = try await store.upsertSamples([consolidatedSample], observedAt: Date(timeIntervalSince1970: 3_060))
478
        try await store.recordDisappearance(sampleUUIDHash: HashService.sampleUUIDHash(firstOldSample.uuid.uuidString), sampleTypeIdentifier: typeIdentifier, observedMissingAt: Date(timeIntervalSince1970: 3_120))
479
        try await store.recordDisappearance(sampleUUIDHash: HashService.sampleUUIDHash(secondOldSample.uuid.uuidString), sampleTypeIdentifier: typeIdentifier, observedMissingAt: Date(timeIntervalSince1970: 3_180))
480

            
481
        let observationIDs = try observationIDs(at: url)
482
        XCTAssertEqual(observationIDs.count, 4)
483

            
484
        let rows = try await store.consolidationEvidence(HealthArchiveConsolidationEvidenceRequest(
485
            fromObservationID: observationIDs[0],
486
            toObservationID: observationIDs[observationIDs.count - 1],
487
            sampleTypeIdentifier: typeIdentifier
488
        ))
489

            
490
        XCTAssertEqual(rows.count, 1)
491
        XCTAssertEqual(rows.first?.sampleTypeIdentifier, typeIdentifier)
492
        XCTAssertEqual(rows.first?.disappearedCount, 2)
493
        XCTAssertEqual(rows.first?.appearedCount, 1)
494
        XCTAssertEqual(rows.first?.fromVisibleRecordCount, 2)
495
        XCTAssertEqual(rows.first?.toVisibleRecordCount, 1)
496
        XCTAssertEqual(rows.first?.fromValueSum, 30)
497
        XCTAssertEqual(rows.first?.toValueSum, 30)
498
        XCTAssertEqual(rows.first?.label, "consolidation_likely")
499
        XCTAssertTrue(rows.first?.sourceCompatible == true)
500
    }
501

            
502
    func testConsolidationEvidenceClassifiesDenseStableShiftAsAggregateChanged() async throws {
503
        let url = databaseURL()
504
        let store = SQLiteHealthArchiveStore(databaseURL: url)
505
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
506

            
507
        let firstOldSample = makeStepCountSample(value: 10, start: 1_000, end: 1_500)
508
        let secondOldSample = makeStepCountSample(value: 20, start: 5_000, end: 6_000)
509
        let denseSample = makeStepCountSample(value: 30, start: 1_000, end: 1_200)
510

            
511
        _ = try await store.upsertSamples([firstOldSample, secondOldSample], observedAt: Date(timeIntervalSince1970: 3_000))
512
        _ = try await store.upsertSamples([denseSample], observedAt: Date(timeIntervalSince1970: 3_060))
513
        try await store.recordDisappearance(sampleUUIDHash: HashService.sampleUUIDHash(firstOldSample.uuid.uuidString), sampleTypeIdentifier: typeIdentifier, observedMissingAt: Date(timeIntervalSince1970: 3_120))
514
        try await store.recordDisappearance(sampleUUIDHash: HashService.sampleUUIDHash(secondOldSample.uuid.uuidString), sampleTypeIdentifier: typeIdentifier, observedMissingAt: Date(timeIntervalSince1970: 3_180))
515

            
516
        let observationIDs = try observationIDs(at: url)
517
        XCTAssertEqual(observationIDs.count, 4)
518

            
519
        let rows = try await store.consolidationEvidence(HealthArchiveConsolidationEvidenceRequest(
520
            fromObservationID: observationIDs[0],
521
            toObservationID: observationIDs[observationIDs.count - 1],
522
            sampleTypeIdentifier: typeIdentifier
523
        ))
524

            
525
        XCTAssertEqual(rows.count, 1)
526
        XCTAssertEqual(rows.first?.label, "aggregate_changed")
527
        XCTAssertEqual(rows.first?.fromVisibleRecordCount, 2)
528
        XCTAssertEqual(rows.first?.toVisibleRecordCount, 1)
529
        XCTAssertEqual(rows.first?.fromValueSum, 30)
530
        XCTAssertEqual(rows.first?.toValueSum, 30)
531
    }
532

            
Bogdan Timofte authored 2 weeks ago
533
    private func databaseURL() -> URL {
534
        temporaryDirectory.appending(path: "Archive.sqlite")
535
    }
536

            
537
    private func createPrototypeDatabase(at url: URL) throws {
538
        var db: OpaquePointer?
539
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
540
            sqlite3_close(db)
541
            XCTFail("Could not create prototype database")
542
            return
543
        }
544
        defer { sqlite3_close(db) }
545
        let status = sqlite3_exec(db, "CREATE TABLE archive_samples (id TEXT PRIMARY KEY)", nil, nil, nil)
546
        XCTAssertEqual(status, SQLITE_OK)
547
    }
548

            
549
    private func makeStepCountSample() -> HKQuantitySample {
Bogdan Timofte authored 2 weeks ago
550
        makeStepCountSample(value: 42, start: 1_000)
551
    }
552

            
Bogdan Timofte authored 2 weeks ago
553
    private func makeStepCountSample(value: Double, start: TimeInterval, end: TimeInterval? = nil) -> HKQuantitySample {
Bogdan Timofte authored 2 weeks ago
554
        let quantityType = HKQuantityType.quantityType(forIdentifier: .stepCount)!
Bogdan Timofte authored 2 weeks ago
555
        let quantity = HKQuantity(unit: .count(), doubleValue: value)
556
        let startDate = Date(timeIntervalSince1970: start)
Bogdan Timofte authored 2 weeks ago
557
        let endDate = Date(timeIntervalSince1970: end ?? (start + 300))
Bogdan Timofte authored 2 weeks ago
558
        return HKQuantitySample(type: quantityType, quantity: quantity, start: startDate, end: endDate)
Bogdan Timofte authored 2 weeks ago
559
    }
560

            
Bogdan Timofte authored 2 weeks ago
561
    private func makeStepCountSamples(count: Int, startIndex: Int) -> [HKQuantitySample] {
562
        (0..<count).map { offset in
563
            let index = startIndex + offset
564
            return makeStepCountSample(
565
                value: Double((index % 97) + 1),
566
                start: 10_000 + Double(index * 600)
567
            )
568
        }
569
    }
570

            
Bogdan Timofte authored 2 weeks ago
571
    private func waitForArchiveOperation<T>(_ operation: @escaping () async throws -> T) throws -> T {
572
        let expectation = expectation(description: "archive operation")
573
        let box = AsyncResultBox<T>()
574

            
575
        Task {
576
            do {
577
                box.result = .success(try await operation())
578
            } catch {
579
                box.result = .failure(error)
580
            }
581
            expectation.fulfill()
582
        }
583

            
584
        wait(for: [expectation], timeout: 20)
585
        return try XCTUnwrap(box.result).get()
586
    }
587

            
Bogdan Timofte authored 2 weeks ago
588
    private func countRows(in tableName: String, at url: URL) throws -> Int {
589
        var db: OpaquePointer?
590
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
591
            sqlite3_close(db)
592
            XCTFail("Could not open test database")
593
            return 0
594
        }
595
        defer { sqlite3_close(db) }
596

            
597
        var statement: OpaquePointer?
598
        guard sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM \(tableName)", -1, &statement, nil) == SQLITE_OK else {
599
            sqlite3_finalize(statement)
600
            XCTFail("Could not prepare count query")
601
            return 0
602
        }
603
        defer { sqlite3_finalize(statement) }
604

            
605
        guard sqlite3_step(statement) == SQLITE_ROW else {
606
            return 0
607
        }
608
        return Int(sqlite3_column_int(statement, 0))
609
    }
610

            
Bogdan Timofte authored 2 weeks ago
611
    private func tableExists(_ tableName: String, at url: URL) throws -> Bool {
612
        var db: OpaquePointer?
613
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
614
            sqlite3_close(db)
615
            XCTFail("Could not open test database")
616
            return false
617
        }
618
        defer { sqlite3_close(db) }
619

            
620
        let sql = """
621
        SELECT 1
622
        FROM sqlite_master
623
        WHERE type = 'table' AND name = ?
624
        LIMIT 1
625
        """
626
        var statement: OpaquePointer?
627
        guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else {
628
            sqlite3_finalize(statement)
629
            XCTFail("Could not prepare table existence query")
630
            return false
631
        }
632
        defer { sqlite3_finalize(statement) }
633

            
634
        sqlite3_bind_text(statement, 1, tableName, -1, unsafeBitCast(-1, to: sqlite3_destructor_type.self))
635
        return sqlite3_step(statement) == SQLITE_ROW
636
    }
637

            
Bogdan Timofte authored 2 weeks ago
638
    private func observationIDs(at url: URL) throws -> [Int64] {
639
        var db: OpaquePointer?
640
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
641
            sqlite3_close(db)
642
            XCTFail("Could not open test database")
643
            return []
644
        }
645
        defer { sqlite3_close(db) }
646

            
647
        var statement: OpaquePointer?
648
        guard sqlite3_prepare_v2(db, "SELECT id FROM observations ORDER BY id", -1, &statement, nil) == SQLITE_OK else {
649
            sqlite3_finalize(statement)
650
            XCTFail("Could not prepare observation query")
651
            return []
652
        }
653
        defer { sqlite3_finalize(statement) }
654

            
655
        var ids: [Int64] = []
656
        while sqlite3_step(statement) == SQLITE_ROW {
657
            ids.append(sqlite3_column_int64(statement, 0))
658
        }
659
        return ids
660
    }
661

            
Bogdan Timofte authored 2 weeks ago
662
    private func sampleVersionDebugRows(at url: URL) throws -> String {
663
        var db: OpaquePointer?
664
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
665
            sqlite3_close(db)
666
            return "could not open database"
667
        }
668
        defer { sqlite3_close(db) }
669

            
670
        let sql = """
671
        SELECT v.payload_hash, v.start_date, v.end_date, v.value_kind, v.numeric_value, v.unit,
672
               v.source_revision_id, src.bundle_identifier, sr.product_type, sr.version,
673
               sr.operating_system_version, v.hk_device_id, v.metadata_id
674
        FROM sample_versions v
675
        LEFT JOIN source_revisions sr ON sr.id = v.source_revision_id
676
        LEFT JOIN sources src ON src.id = sr.source_id
677
        ORDER BY v.id
678
        """
679
        var statement: OpaquePointer?
680
        guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else {
681
            sqlite3_finalize(statement)
682
            return "could not prepare version debug query"
683
        }
684
        defer { sqlite3_finalize(statement) }
685

            
686
        var rows: [String] = []
687
        while sqlite3_step(statement) == SQLITE_ROW {
688
            rows.append((0..<13).map { index in
689
                if sqlite3_column_type(statement, Int32(index)) == SQLITE_NULL {
690
                    return "null"
691
                }
692
                if let text = sqlite3_column_text(statement, Int32(index)) {
693
                    return String(cString: text)
694
                }
695
                return "\(sqlite3_column_double(statement, Int32(index)))"
696
            }.joined(separator: "|"))
697
        }
698
        return rows.joined(separator: "\n")
699
    }
Bogdan Timofte authored a week ago
700

            
701
    private func visibilityRangeDebugRows(at url: URL) throws -> String {
702
        var db: OpaquePointer?
703
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
704
            sqlite3_close(db)
705
            return "could not open database"
706
        }
707
        defer { sqlite3_close(db) }
708

            
709
        let sql = """
710
        SELECT sample_id, version_id, first_observation_id, last_observation_id, first_seen_at, last_seen_at
711
        FROM sample_visibility_ranges
712
        ORDER BY sample_id, version_id, first_observation_id
713
        """
714
        var statement: OpaquePointer?
715
        guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else {
716
            sqlite3_finalize(statement)
717
            return "could not prepare visibility debug query"
718
        }
719
        defer { sqlite3_finalize(statement) }
720

            
721
        var rows: [String] = []
722
        while sqlite3_step(statement) == SQLITE_ROW {
723
            rows.append((0..<6).map { index in
724
                if sqlite3_column_type(statement, Int32(index)) == SQLITE_NULL {
725
                    return "null"
726
                }
727
                if let text = sqlite3_column_text(statement, Int32(index)) {
728
                    return String(cString: text)
729
                }
730
                return "\(sqlite3_column_double(statement, Int32(index)))"
731
            }.joined(separator: "|"))
732
        }
733
        return rows.joined(separator: "\n")
734
    }
Bogdan Timofte authored 2 weeks ago
735
}