HealthProbe / HealthProbeTests / SQLiteHealthArchiveStoreTests.swift
Newer Older
731 lines | 34.69kb
Bogdan Timofte authored 2 weeks ago
1
import HealthKit
2
import SQLite3
3
import XCTest
4
@testable import HealthProbe
5

            
6
final class SQLiteHealthArchiveStoreTests: XCTestCase {
Bogdan Timofte authored 2 weeks ago
7
    private final class AsyncResultBox<T>: @unchecked Sendable {
8
        var result: Result<T, Error>?
9
    }
10

            
Bogdan Timofte authored 2 weeks ago
11
    private var temporaryDirectory: URL!
12

            
13
    override func setUpWithError() throws {
14
        temporaryDirectory = FileManager.default.temporaryDirectory
15
            .appending(path: "HealthProbeTests-\(UUID().uuidString)", directoryHint: .isDirectory)
16
        try FileManager.default.createDirectory(at: temporaryDirectory, withIntermediateDirectories: true)
17
    }
18

            
19
    override func tearDownWithError() throws {
20
        if let temporaryDirectory {
21
            try? FileManager.default.removeItem(at: temporaryDirectory)
22
        }
23
        temporaryDirectory = nil
24
    }
25

            
26
    func testFreshArchiveInitializesSchemaAndPassesIntegrity() async throws {
27
        let store = SQLiteHealthArchiveStore(databaseURL: databaseURL())
28

            
29
        let report = try await store.checkIntegrity()
30
        let records = try await store.records(for: HealthArchiveRecordRequest(
31
            sampleTypeIdentifier: HKQuantityTypeIdentifier.stepCount.rawValue
32
        ))
33

            
34
        XCTAssertTrue(report.passed)
35
        XCTAssertEqual(report.schemaVersion, 2)
36
        XCTAssertEqual(report.sqliteIntegrityStatus, "ok")
37
        XCTAssertEqual(report.foreignKeyIssueCount, 0)
38
        XCTAssertTrue(report.missingTableNames.isEmpty)
39
        XCTAssertTrue(report.requiredTableNames.contains("sample_visibility_ranges"))
40
        XCTAssertTrue(report.requiredTableNames.contains("daily_type_aggregates"))
41
        XCTAssertTrue(records.isEmpty)
42
    }
43

            
44
    func testPrototypeArchiveIsResetAndReinitializedAsV2() async throws {
45
        let url = databaseURL()
46
        try createPrototypeDatabase(at: url)
47
        let store = SQLiteHealthArchiveStore(databaseURL: url)
48

            
49
        let report = try await store.checkIntegrity()
50

            
51
        XCTAssertTrue(report.passed)
52
        XCTAssertEqual(report.schemaVersion, 2)
53
        XCTAssertTrue(report.missingTableNames.isEmpty)
54
    }
55

            
56
    func testRepeatedSamplePageDoesNotDuplicateIdentityOrVersion() async throws {
57
        let url = databaseURL()
58
        let store = SQLiteHealthArchiveStore(databaseURL: url)
59
        let sample = makeStepCountSample()
60

            
61
        let firstWrite = try await store.upsertSamples([sample], observedAt: Date(timeIntervalSince1970: 2_000))
62
        let secondWrite = try await store.upsertSamples([sample], observedAt: Date(timeIntervalSince1970: 2_060))
63
        let records = try await store.records(for: HealthArchiveRecordRequest(
64
            sampleTypeIdentifier: sample.sampleType.identifier
65
        ))
66
        let report = try await store.checkIntegrity()
67
        let versionDebugRows = try sampleVersionDebugRows(at: url)
Bogdan Timofte authored a week ago
68
        let visibilityDebugRows = try visibilityRangeDebugRows(at: url)
Bogdan Timofte authored 2 weeks ago
69

            
70
        XCTAssertEqual(firstWrite.insertedCount, 1)
71
        XCTAssertEqual(firstWrite.updatedCount, 0)
72
        XCTAssertEqual(firstWrite.unchangedCount, 0)
73
        XCTAssertEqual(try countRows(in: "samples", at: url), 1)
74
        XCTAssertEqual(try countRows(in: "sample_versions", at: url), 1, versionDebugRows)
Bogdan Timofte authored a week ago
75
        XCTAssertEqual(try countRows(in: "sample_visibility_ranges", at: url), 1, visibilityDebugRows)
Bogdan Timofte authored 2 weeks ago
76
        XCTAssertEqual(try countRows(in: "source_revisions", at: url), 1)
Bogdan Timofte authored 2 weeks ago
77
        XCTAssertFalse(try tableExists("archive_samples", at: url))
Bogdan Timofte authored 2 weeks ago
78
        XCTAssertEqual(secondWrite.insertedCount, 0)
79
        XCTAssertEqual(secondWrite.updatedCount, 0)
80
        XCTAssertEqual(secondWrite.unchangedCount, 1)
81
        XCTAssertEqual(records.count, 1)
82
        XCTAssertEqual(records.first?.displayValue, "42.0 count")
83
        XCTAssertTrue(report.passed)
84
    }
85

            
Bogdan Timofte authored 2 weeks ago
86
    func testVerificationUsesArchiveV2TablesWithoutLegacyMirror() async throws {
87
        let url = databaseURL()
88
        let store = SQLiteHealthArchiveStore(databaseURL: url)
89
        let sample = makeStepCountSample()
90

            
91
        _ = try await store.upsertSamples([sample], observedAt: Date(timeIntervalSince1970: 2_000))
92
        try await store.markVerification(sampleType: sample.sampleType, verifiedAt: Date(timeIntervalSince1970: 2_060))
93
        let observationIDs = try observationIDs(at: url)
94

            
95
        XCTAssertEqual(observationIDs.count, 2)
Bogdan Timofte authored 2 weeks ago
96
        XCTAssertEqual(try countRows(in: "observation_type_runs", at: url), 2)
97
        XCTAssertEqual(try countRows(in: "observation_type_runs WHERE observation_id = \(observationIDs[0]) AND inserted_event_count = 1", at: url), 1)
Bogdan Timofte authored 2 weeks ago
98
        XCTAssertEqual(try countRows(in: "observation_type_summaries WHERE observation_id = \(observationIDs[1]) AND visible_record_count = 1", at: url), 1)
99
        XCTAssertFalse(try tableExists("archive_samples", at: url))
100
    }
101

            
Bogdan Timofte authored 2 weeks ago
102
    func testDiffSummaryAndRecordsBetweenObservationsUseSQLVisibility() async throws {
103
        let url = databaseURL()
104
        let store = SQLiteHealthArchiveStore(databaseURL: url)
105
        let firstSample = makeStepCountSample(value: 42, start: 1_000)
106
        let secondSample = makeStepCountSample(value: 7, start: 2_000)
107
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
108

            
109
        _ = try await store.upsertSamples([firstSample], observedAt: Date(timeIntervalSince1970: 3_000))
110
        _ = try await store.upsertSamples([firstSample, secondSample], observedAt: Date(timeIntervalSince1970: 3_060))
111
        try await store.recordDisappearance(
112
            sampleUUIDHash: HashService.sampleUUIDHash(firstSample.uuid.uuidString),
113
            sampleTypeIdentifier: typeIdentifier,
114
            observedMissingAt: Date(timeIntervalSince1970: 3_120)
115
        )
116
        let observationIDs = try observationIDs(at: url)
117
        XCTAssertEqual(observationIDs.count, 3)
118

            
119
        let appearedSummary = try await store.diffSummary(HealthArchiveDiffRequest(
120
            fromObservationID: observationIDs[0],
121
            toObservationID: observationIDs[1],
122
            sampleTypeIdentifier: typeIdentifier
123
        ))
124
        let appearedRecords = try await store.diffRecords(HealthArchiveDiffRecordRequest(
125
            fromObservationID: observationIDs[0],
126
            toObservationID: observationIDs[1],
127
            sampleTypeIdentifier: typeIdentifier,
128
            kind: .appeared,
129
            limit: 10
130
        ))
131
        let disappearedSummary = try await store.diffSummary(HealthArchiveDiffRequest(
132
            fromObservationID: observationIDs[1],
133
            toObservationID: observationIDs[2],
134
            sampleTypeIdentifier: typeIdentifier
135
        ))
136
        let disappearedRecords = try await store.diffRecords(HealthArchiveDiffRecordRequest(
137
            fromObservationID: observationIDs[1],
138
            toObservationID: observationIDs[2],
139
            sampleTypeIdentifier: typeIdentifier,
140
            kind: .disappeared,
141
            limit: 10
142
        ))
143

            
144
        XCTAssertEqual(appearedSummary.appearedCount, 1)
145
        XCTAssertEqual(appearedSummary.disappearedCount, 0)
146
        XCTAssertEqual(appearedSummary.representationChangedCount, 0)
147
        XCTAssertEqual(appearedRecords.map(\.displayValue), ["7.0 count"])
148
        XCTAssertEqual(disappearedSummary.appearedCount, 0)
149
        XCTAssertEqual(disappearedSummary.disappearedCount, 1)
150
        XCTAssertEqual(disappearedSummary.representationChangedCount, 0)
151
        XCTAssertEqual(disappearedRecords.map(\.displayValue), ["42.0 count"])
152
        XCTAssertNotNil(disappearedRecords.first?.disappearedAt)
Bogdan Timofte authored 2 weeks ago
153

            
154
        let exportRequest = HealthArchiveReportRequest(
155
            reportID: UUID(),
156
            title: "Disappeared Step Count Export",
157
            typeIdentifierFilter: typeIdentifier,
158
            diffFromObservationID: observationIDs[1],
159
            diffToObservationID: observationIDs[2],
160
            diffKind: .disappeared
161
        )
162
        let exportPreview = try await store.exportPreview(exportRequest)
163
        let exportURL = try await store.exportReport(exportRequest)
164

            
165
        XCTAssertEqual(exportPreview.estimatedRecordCount, 1)
166
        XCTAssertTrue(FileManager.default.fileExists(atPath: exportURL.path))
167
        XCTAssertEqual(try countRows(in: "export_manifests WHERE record_count = 1 AND from_observation_id = \(observationIDs[1]) AND to_observation_id = \(observationIDs[2])", at: url), 1)
168
    }
169

            
170
    func testExportPreviewAndReportUseSQLiteManifest() async throws {
171
        let url = databaseURL()
172
        let store = SQLiteHealthArchiveStore(databaseURL: url)
173
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
174
        let samples = [
175
            makeStepCountSample(value: 42, start: 1_000),
176
            makeStepCountSample(value: 7, start: 2_000)
177
        ]
178

            
179
        _ = try await store.upsertSamples(samples, observedAt: Date(timeIntervalSince1970: 3_000))
180

            
181
        let request = HealthArchiveReportRequest(
182
            reportID: UUID(),
183
            title: "Step Count Export",
184
            typeIdentifierFilter: typeIdentifier
185
        )
186
        let preview = try await store.exportPreview(request)
187
        let exportURL = try await store.exportReport(request)
188

            
189
        XCTAssertEqual(preview.estimatedRecordCount, 2)
190
        XCTAssertTrue(FileManager.default.fileExists(atPath: exportURL.path))
191
        XCTAssertEqual(try countRows(in: "export_manifests", at: url), 1)
192
        XCTAssertEqual(try countRows(in: "export_manifests WHERE record_count = 2", at: url), 1)
193

            
194
        let cache = try CoreDataArchiveCacheStore(inMemory: true)
195
        let rebuild = try cache.rebuild(fromArchiveAt: url)
196
        XCTAssertEqual(rebuild.exportManifestRows, 1)
Bogdan Timofte authored 2 weeks ago
197
    }
198

            
Bogdan Timofte authored 2 weeks ago
199
    func testGroupedObservationKeepsPageWritesDeletesAndVerificationTogether() async throws {
200
        let url = databaseURL()
201
        let store = SQLiteHealthArchiveStore(databaseURL: url)
202
        let firstSample = makeStepCountSample(value: 42, start: 1_000)
203
        let secondSample = makeStepCountSample(value: 7, start: 2_000)
204
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
205

            
206
        _ = try await store.upsertSamples([firstSample], observedAt: Date(timeIntervalSince1970: 3_000))
207
        let observationID = try await store.beginObservation(
208
            observedAt: Date(timeIntervalSince1970: 3_060),
209
            triggerReason: "manual",
210
            selectedTypeSetHash: "selected-types"
211
        )
212
        _ = try await store.upsertSamples(
213
            [secondSample],
214
            observedAt: Date(timeIntervalSince1970: 3_060),
215
            observationID: observationID
216
        )
217
        try await store.recordDisappearance(
218
            sampleUUIDHash: HashService.sampleUUIDHash(firstSample.uuid.uuidString),
219
            sampleTypeIdentifier: typeIdentifier,
220
            observedMissingAt: Date(timeIntervalSince1970: 3_060),
221
            observationID: observationID
222
        )
Bogdan Timofte authored a week ago
223
        XCTAssertEqual(
224
            try countRows(
225
                in: "observation_type_summaries WHERE observation_id = \(observationID)",
226
                at: url
227
            ),
228
            0
229
        )
230
        XCTAssertEqual(
231
            try countRows(
232
                in: "daily_type_aggregates WHERE observation_id = \(observationID)",
233
                at: url
234
            ),
235
            0
236
        )
Bogdan Timofte authored 2 weeks ago
237
        try await store.markVerification(
238
            sampleType: secondSample.sampleType,
239
            verifiedAt: Date(timeIntervalSince1970: 3_060),
240
            observationID: observationID
241
        )
242
        try await store.finishObservation(
243
            observationID: observationID,
244
            status: "completed",
245
            endedAt: Date(timeIntervalSince1970: 3_070)
246
        )
247

            
248
        let observationIDs = try observationIDs(at: url)
249
        let summary = try await store.diffSummary(HealthArchiveDiffRequest(
250
            fromObservationID: observationIDs[0],
251
            toObservationID: observationID,
252
            sampleTypeIdentifier: typeIdentifier
253
        ))
254

            
255
        XCTAssertEqual(observationIDs.count, 2)
256
        XCTAssertEqual(summary.appearedCount, 1)
257
        XCTAssertEqual(summary.disappearedCount, 1)
258
        XCTAssertEqual(try countRows(in: "observations WHERE id = \(observationID) AND status = 'completed' AND selected_type_set_hash = 'selected-types'", at: url), 1)
259
        XCTAssertEqual(try countRows(in: "sample_observation_events WHERE observation_id = \(observationID)", at: url), 2)
260
        XCTAssertEqual(try countRows(in: "observation_type_runs WHERE observation_id = \(observationID) AND inserted_event_count = 1 AND deleted_event_count = 1 AND verified_visible_count = 1 AND status = 'completed'", at: url), 1)
Bogdan Timofte authored a week ago
261
        XCTAssertEqual(try countRows(in: "observation_type_summaries WHERE observation_id = \(observationID) AND visible_record_count = 1", at: url), 1)
262
        XCTAssertGreaterThan(try countRows(in: "daily_type_aggregates WHERE observation_id = \(observationID)", at: url), 0)
Bogdan Timofte authored 2 weeks ago
263
    }
264

            
Bogdan Timofte authored a week ago
265
    func testGroupedObservationCanBatchDeletedObjectsInSingleRun() async throws {
266
        let url = databaseURL()
267
        let store = SQLiteHealthArchiveStore(databaseURL: url)
268
        let samples = [
269
            makeStepCountSample(value: 42, start: 1_000),
270
            makeStepCountSample(value: 7, start: 2_000),
271
            makeStepCountSample(value: 9, start: 3_000)
272
        ]
273
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
274

            
275
        _ = try await store.upsertSamples(samples, observedAt: Date(timeIntervalSince1970: 4_000))
276
        let observationID = try await store.beginObservation(
277
            observedAt: Date(timeIntervalSince1970: 4_060),
278
            triggerReason: "manual",
279
            selectedTypeSetHash: "selected-types"
280
        )
281
        let deletedCount = try await store.recordDisappearances(
282
            sampleUUIDHashes: samples.map { HashService.sampleUUIDHash($0.uuid.uuidString) },
283
            sampleTypeIdentifier: typeIdentifier,
284
            observedMissingAt: Date(timeIntervalSince1970: 4_060),
285
            observationID: observationID
286
        )
287
        try await store.markVerification(
288
            sampleType: samples[0].sampleType,
289
            verifiedAt: Date(timeIntervalSince1970: 4_060),
290
            observationID: observationID
291
        )
292

            
293
        XCTAssertEqual(deletedCount, 3)
294
        XCTAssertEqual(try countRows(in: "sample_observation_events WHERE observation_id = \(observationID) AND event_kind = 'disappeared'", at: url), 3)
295
        XCTAssertEqual(try countRows(in: "observation_type_runs WHERE observation_id = \(observationID) AND deleted_event_count = 3 AND status = 'completed'", at: url), 1)
296
    }
297

            
Bogdan Timofte authored 2 weeks ago
298
    func testLargeSyntheticDiffReturnsCountsAndBoundedPages() async throws {
299
        let url = databaseURL()
300
        let store = SQLiteHealthArchiveStore(databaseURL: url)
301
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
302
        let initialCount = 1_200
303
        let appearedCount = 180
304
        let disappearedCount = 160
305
        let pageSize = 25
306
        let initialSamples = makeStepCountSamples(count: initialCount, startIndex: 0)
307
        let appearedSamples = makeStepCountSamples(count: appearedCount, startIndex: initialCount)
308

            
309
        _ = try await store.upsertSamples(initialSamples, observedAt: Date(timeIntervalSince1970: 1_000_000))
310
        _ = try await store.upsertSamples(initialSamples + appearedSamples, observedAt: Date(timeIntervalSince1970: 1_000_060))
311
        for (offset, sample) in initialSamples.prefix(disappearedCount).enumerated() {
312
            try await store.recordDisappearance(
313
                sampleUUIDHash: HashService.sampleUUIDHash(sample.uuid.uuidString),
314
                sampleTypeIdentifier: typeIdentifier,
315
                observedMissingAt: Date(timeIntervalSince1970: 1_000_120 + Double(offset))
316
            )
317
        }
318

            
319
        let observationIDs = try observationIDs(at: url)
320
        let firstObservationID = try XCTUnwrap(observationIDs.first)
321
        let lastObservationID = try XCTUnwrap(observationIDs.last)
322
        let queryStartedAt = Date()
323
        let summary = try await store.diffSummary(HealthArchiveDiffRequest(
324
            fromObservationID: firstObservationID,
325
            toObservationID: lastObservationID,
326
            sampleTypeIdentifier: typeIdentifier
327
        ))
328
        let firstPage = try await store.diffRecords(HealthArchiveDiffRecordRequest(
329
            fromObservationID: firstObservationID,
330
            toObservationID: lastObservationID,
331
            sampleTypeIdentifier: typeIdentifier,
332
            kind: .appeared,
333
            limit: pageSize
334
        ))
335
        let firstPageLastRecord = try XCTUnwrap(firstPage.last)
336
        let secondPage = try await store.diffRecords(HealthArchiveDiffRecordRequest(
337
            fromObservationID: firstObservationID,
338
            toObservationID: lastObservationID,
339
            sampleTypeIdentifier: typeIdentifier,
340
            kind: .appeared,
341
            afterCursor: RecordCursor(
342
                startDate: firstPageLastRecord.startDate,
343
                strictFingerprint: firstPageLastRecord.strictFingerprint
344
            ),
345
            limit: pageSize
346
        ))
347
        let queryElapsedSeconds = Date().timeIntervalSince(queryStartedAt)
348

            
349
        XCTAssertEqual(summary.appearedCount, appearedCount)
350
        XCTAssertEqual(summary.disappearedCount, disappearedCount)
351
        XCTAssertEqual(summary.representationChangedCount, 0)
352
        XCTAssertEqual(firstPage.count, pageSize)
353
        XCTAssertEqual(secondPage.count, pageSize)
354
        XCTAssertTrue(Set(firstPage.map(\.id)).isDisjoint(with: Set(secondPage.map(\.id))))
355
        XCTAssertLessThan(queryElapsedSeconds, 10)
356
    }
357

            
Bogdan Timofte authored 2 weeks ago
358
    func testLargeSyntheticDiffQueryMetrics() throws {
359
        let url = databaseURL()
360
        let store = SQLiteHealthArchiveStore(databaseURL: url)
361
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
362
        let initialCount = 900
363
        let appearedCount = 120
364
        let disappearedCount = 100
365
        let pageSize = 25
366
        let initialSamples = makeStepCountSamples(count: initialCount, startIndex: 0)
367
        let appearedSamples = makeStepCountSamples(count: appearedCount, startIndex: initialCount)
368

            
369
        try waitForArchiveOperation {
370
            _ = try await store.upsertSamples(initialSamples, observedAt: Date(timeIntervalSince1970: 1_200_000))
371
            _ = try await store.upsertSamples(initialSamples + appearedSamples, observedAt: Date(timeIntervalSince1970: 1_200_060))
372
            for (offset, sample) in initialSamples.prefix(disappearedCount).enumerated() {
373
                try await store.recordDisappearance(
374
                    sampleUUIDHash: HashService.sampleUUIDHash(sample.uuid.uuidString),
375
                    sampleTypeIdentifier: typeIdentifier,
376
                    observedMissingAt: Date(timeIntervalSince1970: 1_200_120 + Double(offset))
377
                )
378
            }
379
        }
380

            
381
        let observationIDs = try observationIDs(at: url)
382
        let firstObservationID = try XCTUnwrap(observationIDs.first)
383
        let lastObservationID = try XCTUnwrap(observationIDs.last)
384
        let options = XCTMeasureOptions()
385
        options.iterationCount = 3
386

            
387
        measure(metrics: [XCTClockMetric(), XCTMemoryMetric()], options: options) {
388
            do {
389
                let result = try waitForArchiveOperation {
390
                    let summary = try await store.diffSummary(HealthArchiveDiffRequest(
391
                        fromObservationID: firstObservationID,
392
                        toObservationID: lastObservationID,
393
                        sampleTypeIdentifier: typeIdentifier
394
                    ))
395
                    let records = try await store.diffRecords(HealthArchiveDiffRecordRequest(
396
                        fromObservationID: firstObservationID,
397
                        toObservationID: lastObservationID,
398
                        sampleTypeIdentifier: typeIdentifier,
399
                        kind: .appeared,
400
                        limit: pageSize
401
                    ))
402
                    return (summary, records)
403
                }
404
                XCTAssertEqual(result.0.appearedCount, appearedCount)
405
                XCTAssertEqual(result.0.disappearedCount, disappearedCount)
406
                XCTAssertEqual(result.1.count, pageSize)
407
            } catch {
408
                XCTFail("Measured archive query failed: \(error)")
409
            }
410
        }
411
    }
412

            
Bogdan Timofte authored 2 weeks ago
413
    func testAggregateComparisonUsesMaterializedDailyAggregates() async throws {
414
        let url = databaseURL()
415
        let store = SQLiteHealthArchiveStore(databaseURL: url)
416
        let firstSample = makeStepCountSample(value: 42, start: 1_000)
417
        let secondSample = makeStepCountSample(value: 7, start: 2_000)
418
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
419

            
420
        _ = try await store.upsertSamples([firstSample], observedAt: Date(timeIntervalSince1970: 3_000))
421
        _ = try await store.upsertSamples([firstSample, secondSample], observedAt: Date(timeIntervalSince1970: 3_060))
422
        let observationIDs = try observationIDs(at: url)
423
        XCTAssertEqual(observationIDs.count, 2)
424

            
425
        let rows = try await store.aggregateComparison(HealthArchiveAggregateComparisonRequest(
426
            fromObservationID: observationIDs[0],
427
            toObservationID: observationIDs[1],
428
            sampleTypeIdentifier: typeIdentifier,
429
            limit: 10
430
        ))
431

            
432
        XCTAssertEqual(rows.count, 1)
433
        XCTAssertEqual(rows.first?.sampleTypeIdentifier, typeIdentifier)
434
        XCTAssertEqual(rows.first?.fromVisibleRecordCount, 1)
435
        XCTAssertEqual(rows.first?.toVisibleRecordCount, 2)
436
        XCTAssertEqual(rows.first?.visibleRecordDelta, 1)
437
        XCTAssertEqual(rows.first?.fromValueSum, 42)
438
        XCTAssertEqual(rows.first?.toValueSum, 49)
439
        XCTAssertEqual(rows.first?.valueSumDelta, 7)
440
    }
441

            
Bogdan Timofte authored 2 weeks ago
442
    func testSourceProvenanceBreakdownReturnsVisibleSourceRows() async throws {
443
        let url = databaseURL()
444
        let store = SQLiteHealthArchiveStore(databaseURL: url)
445
        let sample = makeStepCountSample(value: 42, start: 1_000)
446
        let typeIdentifier = sample.sampleType.identifier
447

            
448
        _ = try await store.upsertSamples([sample], observedAt: Date(timeIntervalSince1970: 3_000))
449

            
450
        let rows = try await store.sourceProvenanceBreakdown(HealthArchiveSourceProvenanceRequest(
451
            visibleAtObservationID: nil,
452
            sampleTypeIdentifier: typeIdentifier,
453
            limit: 10
454
        ))
455

            
456
        XCTAssertEqual(rows.count, 1)
457
        XCTAssertEqual(rows.first?.sampleTypeIdentifier, typeIdentifier)
458
        XCTAssertEqual(rows.first?.visibleRecordCount, 1)
459
        XCTAssertEqual(rows.first?.valueSum, 42)
460
        XCTAssertEqual(rows.first?.sourceBundleIdentifier, sample.sourceRevision.source.bundleIdentifier)
461
    }
462

            
463
    func testConsolidationEvidenceClassifiesStableCountDropAsConsolidationLikely() async throws {
464
        let url = databaseURL()
465
        let store = SQLiteHealthArchiveStore(databaseURL: url)
466
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
467

            
468
        let firstOldSample = makeStepCountSample(value: 10, start: 1_000, end: 1_500)
469
        let secondOldSample = makeStepCountSample(value: 20, start: 1_600, end: 2_000)
470
        let consolidatedSample = makeStepCountSample(value: 30, start: 1_000, end: 2_500)
471

            
472
        _ = try await store.upsertSamples([firstOldSample, secondOldSample], observedAt: Date(timeIntervalSince1970: 3_000))
473
        _ = try await store.upsertSamples([consolidatedSample], observedAt: Date(timeIntervalSince1970: 3_060))
474
        try await store.recordDisappearance(sampleUUIDHash: HashService.sampleUUIDHash(firstOldSample.uuid.uuidString), sampleTypeIdentifier: typeIdentifier, observedMissingAt: Date(timeIntervalSince1970: 3_120))
475
        try await store.recordDisappearance(sampleUUIDHash: HashService.sampleUUIDHash(secondOldSample.uuid.uuidString), sampleTypeIdentifier: typeIdentifier, observedMissingAt: Date(timeIntervalSince1970: 3_180))
476

            
477
        let observationIDs = try observationIDs(at: url)
478
        XCTAssertEqual(observationIDs.count, 4)
479

            
480
        let rows = try await store.consolidationEvidence(HealthArchiveConsolidationEvidenceRequest(
481
            fromObservationID: observationIDs[0],
482
            toObservationID: observationIDs[observationIDs.count - 1],
483
            sampleTypeIdentifier: typeIdentifier
484
        ))
485

            
486
        XCTAssertEqual(rows.count, 1)
487
        XCTAssertEqual(rows.first?.sampleTypeIdentifier, typeIdentifier)
488
        XCTAssertEqual(rows.first?.disappearedCount, 2)
489
        XCTAssertEqual(rows.first?.appearedCount, 1)
490
        XCTAssertEqual(rows.first?.fromVisibleRecordCount, 2)
491
        XCTAssertEqual(rows.first?.toVisibleRecordCount, 1)
492
        XCTAssertEqual(rows.first?.fromValueSum, 30)
493
        XCTAssertEqual(rows.first?.toValueSum, 30)
494
        XCTAssertEqual(rows.first?.label, "consolidation_likely")
495
        XCTAssertTrue(rows.first?.sourceCompatible == true)
496
    }
497

            
498
    func testConsolidationEvidenceClassifiesDenseStableShiftAsAggregateChanged() async throws {
499
        let url = databaseURL()
500
        let store = SQLiteHealthArchiveStore(databaseURL: url)
501
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
502

            
503
        let firstOldSample = makeStepCountSample(value: 10, start: 1_000, end: 1_500)
504
        let secondOldSample = makeStepCountSample(value: 20, start: 5_000, end: 6_000)
505
        let denseSample = makeStepCountSample(value: 30, start: 1_000, end: 1_200)
506

            
507
        _ = try await store.upsertSamples([firstOldSample, secondOldSample], observedAt: Date(timeIntervalSince1970: 3_000))
508
        _ = try await store.upsertSamples([denseSample], observedAt: Date(timeIntervalSince1970: 3_060))
509
        try await store.recordDisappearance(sampleUUIDHash: HashService.sampleUUIDHash(firstOldSample.uuid.uuidString), sampleTypeIdentifier: typeIdentifier, observedMissingAt: Date(timeIntervalSince1970: 3_120))
510
        try await store.recordDisappearance(sampleUUIDHash: HashService.sampleUUIDHash(secondOldSample.uuid.uuidString), sampleTypeIdentifier: typeIdentifier, observedMissingAt: Date(timeIntervalSince1970: 3_180))
511

            
512
        let observationIDs = try observationIDs(at: url)
513
        XCTAssertEqual(observationIDs.count, 4)
514

            
515
        let rows = try await store.consolidationEvidence(HealthArchiveConsolidationEvidenceRequest(
516
            fromObservationID: observationIDs[0],
517
            toObservationID: observationIDs[observationIDs.count - 1],
518
            sampleTypeIdentifier: typeIdentifier
519
        ))
520

            
521
        XCTAssertEqual(rows.count, 1)
522
        XCTAssertEqual(rows.first?.label, "aggregate_changed")
523
        XCTAssertEqual(rows.first?.fromVisibleRecordCount, 2)
524
        XCTAssertEqual(rows.first?.toVisibleRecordCount, 1)
525
        XCTAssertEqual(rows.first?.fromValueSum, 30)
526
        XCTAssertEqual(rows.first?.toValueSum, 30)
527
    }
528

            
Bogdan Timofte authored 2 weeks ago
529
    private func databaseURL() -> URL {
530
        temporaryDirectory.appending(path: "Archive.sqlite")
531
    }
532

            
533
    private func createPrototypeDatabase(at url: URL) throws {
534
        var db: OpaquePointer?
535
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
536
            sqlite3_close(db)
537
            XCTFail("Could not create prototype database")
538
            return
539
        }
540
        defer { sqlite3_close(db) }
541
        let status = sqlite3_exec(db, "CREATE TABLE archive_samples (id TEXT PRIMARY KEY)", nil, nil, nil)
542
        XCTAssertEqual(status, SQLITE_OK)
543
    }
544

            
545
    private func makeStepCountSample() -> HKQuantitySample {
Bogdan Timofte authored 2 weeks ago
546
        makeStepCountSample(value: 42, start: 1_000)
547
    }
548

            
Bogdan Timofte authored 2 weeks ago
549
    private func makeStepCountSample(value: Double, start: TimeInterval, end: TimeInterval? = nil) -> HKQuantitySample {
Bogdan Timofte authored 2 weeks ago
550
        let quantityType = HKQuantityType.quantityType(forIdentifier: .stepCount)!
Bogdan Timofte authored 2 weeks ago
551
        let quantity = HKQuantity(unit: .count(), doubleValue: value)
552
        let startDate = Date(timeIntervalSince1970: start)
Bogdan Timofte authored 2 weeks ago
553
        let endDate = Date(timeIntervalSince1970: end ?? (start + 300))
Bogdan Timofte authored 2 weeks ago
554
        return HKQuantitySample(type: quantityType, quantity: quantity, start: startDate, end: endDate)
Bogdan Timofte authored 2 weeks ago
555
    }
556

            
Bogdan Timofte authored 2 weeks ago
557
    private func makeStepCountSamples(count: Int, startIndex: Int) -> [HKQuantitySample] {
558
        (0..<count).map { offset in
559
            let index = startIndex + offset
560
            return makeStepCountSample(
561
                value: Double((index % 97) + 1),
562
                start: 10_000 + Double(index * 600)
563
            )
564
        }
565
    }
566

            
Bogdan Timofte authored 2 weeks ago
567
    private func waitForArchiveOperation<T>(_ operation: @escaping () async throws -> T) throws -> T {
568
        let expectation = expectation(description: "archive operation")
569
        let box = AsyncResultBox<T>()
570

            
571
        Task {
572
            do {
573
                box.result = .success(try await operation())
574
            } catch {
575
                box.result = .failure(error)
576
            }
577
            expectation.fulfill()
578
        }
579

            
580
        wait(for: [expectation], timeout: 20)
581
        return try XCTUnwrap(box.result).get()
582
    }
583

            
Bogdan Timofte authored 2 weeks ago
584
    private func countRows(in tableName: String, at url: URL) throws -> Int {
585
        var db: OpaquePointer?
586
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
587
            sqlite3_close(db)
588
            XCTFail("Could not open test database")
589
            return 0
590
        }
591
        defer { sqlite3_close(db) }
592

            
593
        var statement: OpaquePointer?
594
        guard sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM \(tableName)", -1, &statement, nil) == SQLITE_OK else {
595
            sqlite3_finalize(statement)
596
            XCTFail("Could not prepare count query")
597
            return 0
598
        }
599
        defer { sqlite3_finalize(statement) }
600

            
601
        guard sqlite3_step(statement) == SQLITE_ROW else {
602
            return 0
603
        }
604
        return Int(sqlite3_column_int(statement, 0))
605
    }
606

            
Bogdan Timofte authored 2 weeks ago
607
    private func tableExists(_ tableName: String, at url: URL) throws -> Bool {
608
        var db: OpaquePointer?
609
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
610
            sqlite3_close(db)
611
            XCTFail("Could not open test database")
612
            return false
613
        }
614
        defer { sqlite3_close(db) }
615

            
616
        let sql = """
617
        SELECT 1
618
        FROM sqlite_master
619
        WHERE type = 'table' AND name = ?
620
        LIMIT 1
621
        """
622
        var statement: OpaquePointer?
623
        guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else {
624
            sqlite3_finalize(statement)
625
            XCTFail("Could not prepare table existence query")
626
            return false
627
        }
628
        defer { sqlite3_finalize(statement) }
629

            
630
        sqlite3_bind_text(statement, 1, tableName, -1, unsafeBitCast(-1, to: sqlite3_destructor_type.self))
631
        return sqlite3_step(statement) == SQLITE_ROW
632
    }
633

            
Bogdan Timofte authored 2 weeks ago
634
    private func observationIDs(at url: URL) throws -> [Int64] {
635
        var db: OpaquePointer?
636
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
637
            sqlite3_close(db)
638
            XCTFail("Could not open test database")
639
            return []
640
        }
641
        defer { sqlite3_close(db) }
642

            
643
        var statement: OpaquePointer?
644
        guard sqlite3_prepare_v2(db, "SELECT id FROM observations ORDER BY id", -1, &statement, nil) == SQLITE_OK else {
645
            sqlite3_finalize(statement)
646
            XCTFail("Could not prepare observation query")
647
            return []
648
        }
649
        defer { sqlite3_finalize(statement) }
650

            
651
        var ids: [Int64] = []
652
        while sqlite3_step(statement) == SQLITE_ROW {
653
            ids.append(sqlite3_column_int64(statement, 0))
654
        }
655
        return ids
656
    }
657

            
Bogdan Timofte authored 2 weeks ago
658
    private func sampleVersionDebugRows(at url: URL) throws -> String {
659
        var db: OpaquePointer?
660
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
661
            sqlite3_close(db)
662
            return "could not open database"
663
        }
664
        defer { sqlite3_close(db) }
665

            
666
        let sql = """
667
        SELECT v.payload_hash, v.start_date, v.end_date, v.value_kind, v.numeric_value, v.unit,
668
               v.source_revision_id, src.bundle_identifier, sr.product_type, sr.version,
669
               sr.operating_system_version, v.hk_device_id, v.metadata_id
670
        FROM sample_versions v
671
        LEFT JOIN source_revisions sr ON sr.id = v.source_revision_id
672
        LEFT JOIN sources src ON src.id = sr.source_id
673
        ORDER BY v.id
674
        """
675
        var statement: OpaquePointer?
676
        guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else {
677
            sqlite3_finalize(statement)
678
            return "could not prepare version debug query"
679
        }
680
        defer { sqlite3_finalize(statement) }
681

            
682
        var rows: [String] = []
683
        while sqlite3_step(statement) == SQLITE_ROW {
684
            rows.append((0..<13).map { index in
685
                if sqlite3_column_type(statement, Int32(index)) == SQLITE_NULL {
686
                    return "null"
687
                }
688
                if let text = sqlite3_column_text(statement, Int32(index)) {
689
                    return String(cString: text)
690
                }
691
                return "\(sqlite3_column_double(statement, Int32(index)))"
692
            }.joined(separator: "|"))
693
        }
694
        return rows.joined(separator: "\n")
695
    }
Bogdan Timofte authored a week ago
696

            
697
    private func visibilityRangeDebugRows(at url: URL) throws -> String {
698
        var db: OpaquePointer?
699
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
700
            sqlite3_close(db)
701
            return "could not open database"
702
        }
703
        defer { sqlite3_close(db) }
704

            
705
        let sql = """
706
        SELECT sample_id, version_id, first_observation_id, last_observation_id, first_seen_at, last_seen_at
707
        FROM sample_visibility_ranges
708
        ORDER BY sample_id, version_id, first_observation_id
709
        """
710
        var statement: OpaquePointer?
711
        guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else {
712
            sqlite3_finalize(statement)
713
            return "could not prepare visibility debug query"
714
        }
715
        defer { sqlite3_finalize(statement) }
716

            
717
        var rows: [String] = []
718
        while sqlite3_step(statement) == SQLITE_ROW {
719
            rows.append((0..<6).map { index in
720
                if sqlite3_column_type(statement, Int32(index)) == SQLITE_NULL {
721
                    return "null"
722
                }
723
                if let text = sqlite3_column_text(statement, Int32(index)) {
724
                    return String(cString: text)
725
                }
726
                return "\(sqlite3_column_double(statement, Int32(index)))"
727
            }.joined(separator: "|"))
728
        }
729
        return rows.joined(separator: "\n")
730
    }
Bogdan Timofte authored 2 weeks ago
731
}