HealthProbe / HealthProbeTests / SQLiteHealthArchiveStoreTests.swift
Newer Older
695 lines | 33.148kb
Bogdan Timofte authored 2 weeks ago
1
import HealthKit
2
import SQLite3
3
import XCTest
4
@testable import HealthProbe
5

            
6
final class SQLiteHealthArchiveStoreTests: XCTestCase {
Bogdan Timofte authored 2 weeks ago
7
    private final class AsyncResultBox<T>: @unchecked Sendable {
8
        var result: Result<T, Error>?
9
    }
10

            
Bogdan Timofte authored 2 weeks ago
11
    private var temporaryDirectory: URL!
12

            
13
    override func setUpWithError() throws {
14
        temporaryDirectory = FileManager.default.temporaryDirectory
15
            .appending(path: "HealthProbeTests-\(UUID().uuidString)", directoryHint: .isDirectory)
16
        try FileManager.default.createDirectory(at: temporaryDirectory, withIntermediateDirectories: true)
17
    }
18

            
19
    override func tearDownWithError() throws {
20
        if let temporaryDirectory {
21
            try? FileManager.default.removeItem(at: temporaryDirectory)
22
        }
23
        temporaryDirectory = nil
24
    }
25

            
26
    func testFreshArchiveInitializesSchemaAndPassesIntegrity() async throws {
27
        let store = SQLiteHealthArchiveStore(databaseURL: databaseURL())
28

            
29
        let report = try await store.checkIntegrity()
30
        let records = try await store.records(for: HealthArchiveRecordRequest(
31
            sampleTypeIdentifier: HKQuantityTypeIdentifier.stepCount.rawValue
32
        ))
33

            
34
        XCTAssertTrue(report.passed)
35
        XCTAssertEqual(report.schemaVersion, 2)
36
        XCTAssertEqual(report.sqliteIntegrityStatus, "ok")
37
        XCTAssertEqual(report.foreignKeyIssueCount, 0)
38
        XCTAssertTrue(report.missingTableNames.isEmpty)
39
        XCTAssertTrue(report.requiredTableNames.contains("sample_visibility_ranges"))
40
        XCTAssertTrue(report.requiredTableNames.contains("daily_type_aggregates"))
41
        XCTAssertTrue(records.isEmpty)
42
    }
43

            
44
    func testPrototypeArchiveIsResetAndReinitializedAsV2() async throws {
45
        let url = databaseURL()
46
        try createPrototypeDatabase(at: url)
47
        let store = SQLiteHealthArchiveStore(databaseURL: url)
48

            
49
        let report = try await store.checkIntegrity()
50

            
51
        XCTAssertTrue(report.passed)
52
        XCTAssertEqual(report.schemaVersion, 2)
53
        XCTAssertTrue(report.missingTableNames.isEmpty)
54
    }
55

            
56
    func testRepeatedSamplePageDoesNotDuplicateIdentityOrVersion() async throws {
57
        let url = databaseURL()
58
        let store = SQLiteHealthArchiveStore(databaseURL: url)
59
        let sample = makeStepCountSample()
60

            
61
        let firstWrite = try await store.upsertSamples([sample], observedAt: Date(timeIntervalSince1970: 2_000))
62
        let secondWrite = try await store.upsertSamples([sample], observedAt: Date(timeIntervalSince1970: 2_060))
63
        let records = try await store.records(for: HealthArchiveRecordRequest(
64
            sampleTypeIdentifier: sample.sampleType.identifier
65
        ))
66
        let report = try await store.checkIntegrity()
67
        let versionDebugRows = try sampleVersionDebugRows(at: url)
68

            
69
        XCTAssertEqual(firstWrite.insertedCount, 1)
70
        XCTAssertEqual(firstWrite.updatedCount, 0)
71
        XCTAssertEqual(firstWrite.unchangedCount, 0)
72
        XCTAssertEqual(try countRows(in: "samples", at: url), 1)
73
        XCTAssertEqual(try countRows(in: "sample_versions", at: url), 1, versionDebugRows)
74
        XCTAssertEqual(try countRows(in: "sample_visibility_ranges", at: url), 1)
75
        XCTAssertEqual(try countRows(in: "source_revisions", at: url), 1)
Bogdan Timofte authored 2 weeks ago
76
        XCTAssertFalse(try tableExists("archive_samples", at: url))
Bogdan Timofte authored 2 weeks ago
77
        XCTAssertEqual(secondWrite.insertedCount, 0)
78
        XCTAssertEqual(secondWrite.updatedCount, 0)
79
        XCTAssertEqual(secondWrite.unchangedCount, 1)
80
        XCTAssertEqual(records.count, 1)
81
        XCTAssertEqual(records.first?.displayValue, "42.0 count")
82
        XCTAssertTrue(report.passed)
83
    }
84

            
Bogdan Timofte authored 2 weeks ago
85
    func testVerificationUsesArchiveV2TablesWithoutLegacyMirror() async throws {
86
        let url = databaseURL()
87
        let store = SQLiteHealthArchiveStore(databaseURL: url)
88
        let sample = makeStepCountSample()
89

            
90
        _ = try await store.upsertSamples([sample], observedAt: Date(timeIntervalSince1970: 2_000))
91
        try await store.markVerification(sampleType: sample.sampleType, verifiedAt: Date(timeIntervalSince1970: 2_060))
92
        let observationIDs = try observationIDs(at: url)
93

            
94
        XCTAssertEqual(observationIDs.count, 2)
Bogdan Timofte authored 2 weeks ago
95
        XCTAssertEqual(try countRows(in: "observation_type_runs", at: url), 2)
96
        XCTAssertEqual(try countRows(in: "observation_type_runs WHERE observation_id = \(observationIDs[0]) AND inserted_event_count = 1", at: url), 1)
Bogdan Timofte authored 2 weeks ago
97
        XCTAssertEqual(try countRows(in: "observation_type_summaries WHERE observation_id = \(observationIDs[1]) AND visible_record_count = 1", at: url), 1)
98
        XCTAssertFalse(try tableExists("archive_samples", at: url))
99
    }
100

            
Bogdan Timofte authored 2 weeks ago
101
    func testDiffSummaryAndRecordsBetweenObservationsUseSQLVisibility() async throws {
102
        let url = databaseURL()
103
        let store = SQLiteHealthArchiveStore(databaseURL: url)
104
        let firstSample = makeStepCountSample(value: 42, start: 1_000)
105
        let secondSample = makeStepCountSample(value: 7, start: 2_000)
106
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
107

            
108
        _ = try await store.upsertSamples([firstSample], observedAt: Date(timeIntervalSince1970: 3_000))
109
        _ = try await store.upsertSamples([firstSample, secondSample], observedAt: Date(timeIntervalSince1970: 3_060))
110
        try await store.recordDisappearance(
111
            sampleUUIDHash: HashService.sampleUUIDHash(firstSample.uuid.uuidString),
112
            sampleTypeIdentifier: typeIdentifier,
113
            observedMissingAt: Date(timeIntervalSince1970: 3_120)
114
        )
115
        let observationIDs = try observationIDs(at: url)
116
        XCTAssertEqual(observationIDs.count, 3)
117

            
118
        let appearedSummary = try await store.diffSummary(HealthArchiveDiffRequest(
119
            fromObservationID: observationIDs[0],
120
            toObservationID: observationIDs[1],
121
            sampleTypeIdentifier: typeIdentifier
122
        ))
123
        let appearedRecords = try await store.diffRecords(HealthArchiveDiffRecordRequest(
124
            fromObservationID: observationIDs[0],
125
            toObservationID: observationIDs[1],
126
            sampleTypeIdentifier: typeIdentifier,
127
            kind: .appeared,
128
            limit: 10
129
        ))
130
        let disappearedSummary = try await store.diffSummary(HealthArchiveDiffRequest(
131
            fromObservationID: observationIDs[1],
132
            toObservationID: observationIDs[2],
133
            sampleTypeIdentifier: typeIdentifier
134
        ))
135
        let disappearedRecords = try await store.diffRecords(HealthArchiveDiffRecordRequest(
136
            fromObservationID: observationIDs[1],
137
            toObservationID: observationIDs[2],
138
            sampleTypeIdentifier: typeIdentifier,
139
            kind: .disappeared,
140
            limit: 10
141
        ))
142

            
143
        XCTAssertEqual(appearedSummary.appearedCount, 1)
144
        XCTAssertEqual(appearedSummary.disappearedCount, 0)
145
        XCTAssertEqual(appearedSummary.representationChangedCount, 0)
146
        XCTAssertEqual(appearedRecords.map(\.displayValue), ["7.0 count"])
147
        XCTAssertEqual(disappearedSummary.appearedCount, 0)
148
        XCTAssertEqual(disappearedSummary.disappearedCount, 1)
149
        XCTAssertEqual(disappearedSummary.representationChangedCount, 0)
150
        XCTAssertEqual(disappearedRecords.map(\.displayValue), ["42.0 count"])
151
        XCTAssertNotNil(disappearedRecords.first?.disappearedAt)
Bogdan Timofte authored a week ago
152

            
153
        let exportRequest = HealthArchiveReportRequest(
154
            reportID: UUID(),
155
            title: "Disappeared Step Count Export",
156
            typeIdentifierFilter: typeIdentifier,
157
            diffFromObservationID: observationIDs[1],
158
            diffToObservationID: observationIDs[2],
159
            diffKind: .disappeared
160
        )
161
        let exportPreview = try await store.exportPreview(exportRequest)
162
        let exportURL = try await store.exportReport(exportRequest)
163

            
164
        XCTAssertEqual(exportPreview.estimatedRecordCount, 1)
165
        XCTAssertTrue(FileManager.default.fileExists(atPath: exportURL.path))
166
        XCTAssertEqual(try countRows(in: "export_manifests WHERE record_count = 1 AND from_observation_id = \(observationIDs[1]) AND to_observation_id = \(observationIDs[2])", at: url), 1)
167
    }
168

            
169
    func testExportPreviewAndReportUseSQLiteManifest() async throws {
170
        let url = databaseURL()
171
        let store = SQLiteHealthArchiveStore(databaseURL: url)
172
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
173
        let samples = [
174
            makeStepCountSample(value: 42, start: 1_000),
175
            makeStepCountSample(value: 7, start: 2_000)
176
        ]
177

            
178
        _ = try await store.upsertSamples(samples, observedAt: Date(timeIntervalSince1970: 3_000))
179

            
180
        let request = HealthArchiveReportRequest(
181
            reportID: UUID(),
182
            title: "Step Count Export",
183
            typeIdentifierFilter: typeIdentifier
184
        )
185
        let preview = try await store.exportPreview(request)
186
        let exportURL = try await store.exportReport(request)
187

            
188
        XCTAssertEqual(preview.estimatedRecordCount, 2)
189
        XCTAssertTrue(FileManager.default.fileExists(atPath: exportURL.path))
190
        XCTAssertEqual(try countRows(in: "export_manifests", at: url), 1)
191
        XCTAssertEqual(try countRows(in: "export_manifests WHERE record_count = 2", at: url), 1)
192

            
193
        let cache = try CoreDataArchiveCacheStore(inMemory: true)
194
        let rebuild = try cache.rebuild(fromArchiveAt: url)
195
        XCTAssertEqual(rebuild.exportManifestRows, 1)
Bogdan Timofte authored 2 weeks ago
196
    }
197

            
Bogdan Timofte authored 2 weeks ago
198
    func testGroupedObservationKeepsPageWritesDeletesAndVerificationTogether() async throws {
199
        let url = databaseURL()
200
        let store = SQLiteHealthArchiveStore(databaseURL: url)
201
        let firstSample = makeStepCountSample(value: 42, start: 1_000)
202
        let secondSample = makeStepCountSample(value: 7, start: 2_000)
203
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
204

            
205
        _ = try await store.upsertSamples([firstSample], observedAt: Date(timeIntervalSince1970: 3_000))
206
        let observationID = try await store.beginObservation(
207
            observedAt: Date(timeIntervalSince1970: 3_060),
208
            triggerReason: "manual",
209
            selectedTypeSetHash: "selected-types"
210
        )
211
        _ = try await store.upsertSamples(
212
            [secondSample],
213
            observedAt: Date(timeIntervalSince1970: 3_060),
214
            observationID: observationID
215
        )
216
        try await store.recordDisappearance(
217
            sampleUUIDHash: HashService.sampleUUIDHash(firstSample.uuid.uuidString),
218
            sampleTypeIdentifier: typeIdentifier,
219
            observedMissingAt: Date(timeIntervalSince1970: 3_060),
220
            observationID: observationID
221
        )
Bogdan Timofte authored a week ago
222
        XCTAssertEqual(
223
            try countRows(
224
                in: "observation_type_summaries WHERE observation_id = \(observationID)",
225
                at: url
226
            ),
227
            0
228
        )
229
        XCTAssertEqual(
230
            try countRows(
231
                in: "daily_type_aggregates WHERE observation_id = \(observationID)",
232
                at: url
233
            ),
234
            0
235
        )
Bogdan Timofte authored 2 weeks ago
236
        try await store.markVerification(
237
            sampleType: secondSample.sampleType,
238
            verifiedAt: Date(timeIntervalSince1970: 3_060),
239
            observationID: observationID
240
        )
241
        try await store.finishObservation(
242
            observationID: observationID,
243
            status: "completed",
244
            endedAt: Date(timeIntervalSince1970: 3_070)
245
        )
246

            
247
        let observationIDs = try observationIDs(at: url)
248
        let summary = try await store.diffSummary(HealthArchiveDiffRequest(
249
            fromObservationID: observationIDs[0],
250
            toObservationID: observationID,
251
            sampleTypeIdentifier: typeIdentifier
252
        ))
253

            
254
        XCTAssertEqual(observationIDs.count, 2)
255
        XCTAssertEqual(summary.appearedCount, 1)
256
        XCTAssertEqual(summary.disappearedCount, 1)
257
        XCTAssertEqual(try countRows(in: "observations WHERE id = \(observationID) AND status = 'completed' AND selected_type_set_hash = 'selected-types'", at: url), 1)
258
        XCTAssertEqual(try countRows(in: "sample_observation_events WHERE observation_id = \(observationID)", at: url), 2)
259
        XCTAssertEqual(try countRows(in: "observation_type_runs WHERE observation_id = \(observationID) AND inserted_event_count = 1 AND deleted_event_count = 1 AND verified_visible_count = 1 AND status = 'completed'", at: url), 1)
Bogdan Timofte authored a week ago
260
        XCTAssertEqual(try countRows(in: "observation_type_summaries WHERE observation_id = \(observationID) AND visible_record_count = 1", at: url), 1)
261
        XCTAssertGreaterThan(try countRows(in: "daily_type_aggregates WHERE observation_id = \(observationID)", at: url), 0)
Bogdan Timofte authored 2 weeks ago
262
    }
263

            
Bogdan Timofte authored a week ago
264
    func testGroupedObservationCanBatchDeletedObjectsInSingleRun() async throws {
265
        let url = databaseURL()
266
        let store = SQLiteHealthArchiveStore(databaseURL: url)
267
        let samples = [
268
            makeStepCountSample(value: 42, start: 1_000),
269
            makeStepCountSample(value: 7, start: 2_000),
270
            makeStepCountSample(value: 9, start: 3_000)
271
        ]
272
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
273

            
274
        _ = try await store.upsertSamples(samples, observedAt: Date(timeIntervalSince1970: 4_000))
275
        let observationID = try await store.beginObservation(
276
            observedAt: Date(timeIntervalSince1970: 4_060),
277
            triggerReason: "manual",
278
            selectedTypeSetHash: "selected-types"
279
        )
280
        let deletedCount = try await store.recordDisappearances(
281
            sampleUUIDHashes: samples.map { HashService.sampleUUIDHash($0.uuid.uuidString) },
282
            sampleTypeIdentifier: typeIdentifier,
283
            observedMissingAt: Date(timeIntervalSince1970: 4_060),
284
            observationID: observationID
285
        )
286
        try await store.markVerification(
287
            sampleType: samples[0].sampleType,
288
            verifiedAt: Date(timeIntervalSince1970: 4_060),
289
            observationID: observationID
290
        )
291

            
292
        XCTAssertEqual(deletedCount, 3)
293
        XCTAssertEqual(try countRows(in: "sample_observation_events WHERE observation_id = \(observationID) AND event_kind = 'disappeared'", at: url), 3)
294
        XCTAssertEqual(try countRows(in: "observation_type_runs WHERE observation_id = \(observationID) AND deleted_event_count = 3 AND status = 'completed'", at: url), 1)
295
    }
296

            
Bogdan Timofte authored 2 weeks ago
297
    func testLargeSyntheticDiffReturnsCountsAndBoundedPages() async throws {
298
        let url = databaseURL()
299
        let store = SQLiteHealthArchiveStore(databaseURL: url)
300
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
301
        let initialCount = 1_200
302
        let appearedCount = 180
303
        let disappearedCount = 160
304
        let pageSize = 25
305
        let initialSamples = makeStepCountSamples(count: initialCount, startIndex: 0)
306
        let appearedSamples = makeStepCountSamples(count: appearedCount, startIndex: initialCount)
307

            
308
        _ = try await store.upsertSamples(initialSamples, observedAt: Date(timeIntervalSince1970: 1_000_000))
309
        _ = try await store.upsertSamples(initialSamples + appearedSamples, observedAt: Date(timeIntervalSince1970: 1_000_060))
310
        for (offset, sample) in initialSamples.prefix(disappearedCount).enumerated() {
311
            try await store.recordDisappearance(
312
                sampleUUIDHash: HashService.sampleUUIDHash(sample.uuid.uuidString),
313
                sampleTypeIdentifier: typeIdentifier,
314
                observedMissingAt: Date(timeIntervalSince1970: 1_000_120 + Double(offset))
315
            )
316
        }
317

            
318
        let observationIDs = try observationIDs(at: url)
319
        let firstObservationID = try XCTUnwrap(observationIDs.first)
320
        let lastObservationID = try XCTUnwrap(observationIDs.last)
321
        let queryStartedAt = Date()
322
        let summary = try await store.diffSummary(HealthArchiveDiffRequest(
323
            fromObservationID: firstObservationID,
324
            toObservationID: lastObservationID,
325
            sampleTypeIdentifier: typeIdentifier
326
        ))
327
        let firstPage = try await store.diffRecords(HealthArchiveDiffRecordRequest(
328
            fromObservationID: firstObservationID,
329
            toObservationID: lastObservationID,
330
            sampleTypeIdentifier: typeIdentifier,
331
            kind: .appeared,
332
            limit: pageSize
333
        ))
334
        let firstPageLastRecord = try XCTUnwrap(firstPage.last)
335
        let secondPage = try await store.diffRecords(HealthArchiveDiffRecordRequest(
336
            fromObservationID: firstObservationID,
337
            toObservationID: lastObservationID,
338
            sampleTypeIdentifier: typeIdentifier,
339
            kind: .appeared,
340
            afterCursor: RecordCursor(
341
                startDate: firstPageLastRecord.startDate,
342
                strictFingerprint: firstPageLastRecord.strictFingerprint
343
            ),
344
            limit: pageSize
345
        ))
346
        let queryElapsedSeconds = Date().timeIntervalSince(queryStartedAt)
347

            
348
        XCTAssertEqual(summary.appearedCount, appearedCount)
349
        XCTAssertEqual(summary.disappearedCount, disappearedCount)
350
        XCTAssertEqual(summary.representationChangedCount, 0)
351
        XCTAssertEqual(firstPage.count, pageSize)
352
        XCTAssertEqual(secondPage.count, pageSize)
353
        XCTAssertTrue(Set(firstPage.map(\.id)).isDisjoint(with: Set(secondPage.map(\.id))))
354
        XCTAssertLessThan(queryElapsedSeconds, 10)
355
    }
356

            
Bogdan Timofte authored 2 weeks ago
357
    func testLargeSyntheticDiffQueryMetrics() throws {
358
        let url = databaseURL()
359
        let store = SQLiteHealthArchiveStore(databaseURL: url)
360
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
361
        let initialCount = 900
362
        let appearedCount = 120
363
        let disappearedCount = 100
364
        let pageSize = 25
365
        let initialSamples = makeStepCountSamples(count: initialCount, startIndex: 0)
366
        let appearedSamples = makeStepCountSamples(count: appearedCount, startIndex: initialCount)
367

            
368
        try waitForArchiveOperation {
369
            _ = try await store.upsertSamples(initialSamples, observedAt: Date(timeIntervalSince1970: 1_200_000))
370
            _ = try await store.upsertSamples(initialSamples + appearedSamples, observedAt: Date(timeIntervalSince1970: 1_200_060))
371
            for (offset, sample) in initialSamples.prefix(disappearedCount).enumerated() {
372
                try await store.recordDisappearance(
373
                    sampleUUIDHash: HashService.sampleUUIDHash(sample.uuid.uuidString),
374
                    sampleTypeIdentifier: typeIdentifier,
375
                    observedMissingAt: Date(timeIntervalSince1970: 1_200_120 + Double(offset))
376
                )
377
            }
378
        }
379

            
380
        let observationIDs = try observationIDs(at: url)
381
        let firstObservationID = try XCTUnwrap(observationIDs.first)
382
        let lastObservationID = try XCTUnwrap(observationIDs.last)
383
        let options = XCTMeasureOptions()
384
        options.iterationCount = 3
385

            
386
        measure(metrics: [XCTClockMetric(), XCTMemoryMetric()], options: options) {
387
            do {
388
                let result = try waitForArchiveOperation {
389
                    let summary = try await store.diffSummary(HealthArchiveDiffRequest(
390
                        fromObservationID: firstObservationID,
391
                        toObservationID: lastObservationID,
392
                        sampleTypeIdentifier: typeIdentifier
393
                    ))
394
                    let records = try await store.diffRecords(HealthArchiveDiffRecordRequest(
395
                        fromObservationID: firstObservationID,
396
                        toObservationID: lastObservationID,
397
                        sampleTypeIdentifier: typeIdentifier,
398
                        kind: .appeared,
399
                        limit: pageSize
400
                    ))
401
                    return (summary, records)
402
                }
403
                XCTAssertEqual(result.0.appearedCount, appearedCount)
404
                XCTAssertEqual(result.0.disappearedCount, disappearedCount)
405
                XCTAssertEqual(result.1.count, pageSize)
406
            } catch {
407
                XCTFail("Measured archive query failed: \(error)")
408
            }
409
        }
410
    }
411

            
Bogdan Timofte authored 2 weeks ago
412
    func testAggregateComparisonUsesMaterializedDailyAggregates() async throws {
413
        let url = databaseURL()
414
        let store = SQLiteHealthArchiveStore(databaseURL: url)
415
        let firstSample = makeStepCountSample(value: 42, start: 1_000)
416
        let secondSample = makeStepCountSample(value: 7, start: 2_000)
417
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
418

            
419
        _ = try await store.upsertSamples([firstSample], observedAt: Date(timeIntervalSince1970: 3_000))
420
        _ = try await store.upsertSamples([firstSample, secondSample], observedAt: Date(timeIntervalSince1970: 3_060))
421
        let observationIDs = try observationIDs(at: url)
422
        XCTAssertEqual(observationIDs.count, 2)
423

            
424
        let rows = try await store.aggregateComparison(HealthArchiveAggregateComparisonRequest(
425
            fromObservationID: observationIDs[0],
426
            toObservationID: observationIDs[1],
427
            sampleTypeIdentifier: typeIdentifier,
428
            limit: 10
429
        ))
430

            
431
        XCTAssertEqual(rows.count, 1)
432
        XCTAssertEqual(rows.first?.sampleTypeIdentifier, typeIdentifier)
433
        XCTAssertEqual(rows.first?.fromVisibleRecordCount, 1)
434
        XCTAssertEqual(rows.first?.toVisibleRecordCount, 2)
435
        XCTAssertEqual(rows.first?.visibleRecordDelta, 1)
436
        XCTAssertEqual(rows.first?.fromValueSum, 42)
437
        XCTAssertEqual(rows.first?.toValueSum, 49)
438
        XCTAssertEqual(rows.first?.valueSumDelta, 7)
439
    }
440

            
Bogdan Timofte authored 2 weeks ago
441
    func testSourceProvenanceBreakdownReturnsVisibleSourceRows() async throws {
442
        let url = databaseURL()
443
        let store = SQLiteHealthArchiveStore(databaseURL: url)
444
        let sample = makeStepCountSample(value: 42, start: 1_000)
445
        let typeIdentifier = sample.sampleType.identifier
446

            
447
        _ = try await store.upsertSamples([sample], observedAt: Date(timeIntervalSince1970: 3_000))
448

            
449
        let rows = try await store.sourceProvenanceBreakdown(HealthArchiveSourceProvenanceRequest(
450
            visibleAtObservationID: nil,
451
            sampleTypeIdentifier: typeIdentifier,
452
            limit: 10
453
        ))
454

            
455
        XCTAssertEqual(rows.count, 1)
456
        XCTAssertEqual(rows.first?.sampleTypeIdentifier, typeIdentifier)
457
        XCTAssertEqual(rows.first?.visibleRecordCount, 1)
458
        XCTAssertEqual(rows.first?.valueSum, 42)
459
        XCTAssertEqual(rows.first?.sourceBundleIdentifier, sample.sourceRevision.source.bundleIdentifier)
460
    }
461

            
462
    func testConsolidationEvidenceClassifiesStableCountDropAsConsolidationLikely() async throws {
463
        let url = databaseURL()
464
        let store = SQLiteHealthArchiveStore(databaseURL: url)
465
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
466

            
467
        let firstOldSample = makeStepCountSample(value: 10, start: 1_000, end: 1_500)
468
        let secondOldSample = makeStepCountSample(value: 20, start: 1_600, end: 2_000)
469
        let consolidatedSample = makeStepCountSample(value: 30, start: 1_000, end: 2_500)
470

            
471
        _ = try await store.upsertSamples([firstOldSample, secondOldSample], observedAt: Date(timeIntervalSince1970: 3_000))
472
        _ = try await store.upsertSamples([consolidatedSample], observedAt: Date(timeIntervalSince1970: 3_060))
473
        try await store.recordDisappearance(sampleUUIDHash: HashService.sampleUUIDHash(firstOldSample.uuid.uuidString), sampleTypeIdentifier: typeIdentifier, observedMissingAt: Date(timeIntervalSince1970: 3_120))
474
        try await store.recordDisappearance(sampleUUIDHash: HashService.sampleUUIDHash(secondOldSample.uuid.uuidString), sampleTypeIdentifier: typeIdentifier, observedMissingAt: Date(timeIntervalSince1970: 3_180))
475

            
476
        let observationIDs = try observationIDs(at: url)
477
        XCTAssertEqual(observationIDs.count, 4)
478

            
479
        let rows = try await store.consolidationEvidence(HealthArchiveConsolidationEvidenceRequest(
480
            fromObservationID: observationIDs[0],
481
            toObservationID: observationIDs[observationIDs.count - 1],
482
            sampleTypeIdentifier: typeIdentifier
483
        ))
484

            
485
        XCTAssertEqual(rows.count, 1)
486
        XCTAssertEqual(rows.first?.sampleTypeIdentifier, typeIdentifier)
487
        XCTAssertEqual(rows.first?.disappearedCount, 2)
488
        XCTAssertEqual(rows.first?.appearedCount, 1)
489
        XCTAssertEqual(rows.first?.fromVisibleRecordCount, 2)
490
        XCTAssertEqual(rows.first?.toVisibleRecordCount, 1)
491
        XCTAssertEqual(rows.first?.fromValueSum, 30)
492
        XCTAssertEqual(rows.first?.toValueSum, 30)
493
        XCTAssertEqual(rows.first?.label, "consolidation_likely")
494
        XCTAssertTrue(rows.first?.sourceCompatible == true)
495
    }
496

            
497
    func testConsolidationEvidenceClassifiesDenseStableShiftAsAggregateChanged() async throws {
498
        let url = databaseURL()
499
        let store = SQLiteHealthArchiveStore(databaseURL: url)
500
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
501

            
502
        let firstOldSample = makeStepCountSample(value: 10, start: 1_000, end: 1_500)
503
        let secondOldSample = makeStepCountSample(value: 20, start: 5_000, end: 6_000)
504
        let denseSample = makeStepCountSample(value: 30, start: 1_000, end: 1_200)
505

            
506
        _ = try await store.upsertSamples([firstOldSample, secondOldSample], observedAt: Date(timeIntervalSince1970: 3_000))
507
        _ = try await store.upsertSamples([denseSample], observedAt: Date(timeIntervalSince1970: 3_060))
508
        try await store.recordDisappearance(sampleUUIDHash: HashService.sampleUUIDHash(firstOldSample.uuid.uuidString), sampleTypeIdentifier: typeIdentifier, observedMissingAt: Date(timeIntervalSince1970: 3_120))
509
        try await store.recordDisappearance(sampleUUIDHash: HashService.sampleUUIDHash(secondOldSample.uuid.uuidString), sampleTypeIdentifier: typeIdentifier, observedMissingAt: Date(timeIntervalSince1970: 3_180))
510

            
511
        let observationIDs = try observationIDs(at: url)
512
        XCTAssertEqual(observationIDs.count, 4)
513

            
514
        let rows = try await store.consolidationEvidence(HealthArchiveConsolidationEvidenceRequest(
515
            fromObservationID: observationIDs[0],
516
            toObservationID: observationIDs[observationIDs.count - 1],
517
            sampleTypeIdentifier: typeIdentifier
518
        ))
519

            
520
        XCTAssertEqual(rows.count, 1)
521
        XCTAssertEqual(rows.first?.label, "aggregate_changed")
522
        XCTAssertEqual(rows.first?.fromVisibleRecordCount, 2)
523
        XCTAssertEqual(rows.first?.toVisibleRecordCount, 1)
524
        XCTAssertEqual(rows.first?.fromValueSum, 30)
525
        XCTAssertEqual(rows.first?.toValueSum, 30)
526
    }
527

            
Bogdan Timofte authored 2 weeks ago
528
    private func databaseURL() -> URL {
529
        temporaryDirectory.appending(path: "Archive.sqlite")
530
    }
531

            
532
    private func createPrototypeDatabase(at url: URL) throws {
533
        var db: OpaquePointer?
534
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
535
            sqlite3_close(db)
536
            XCTFail("Could not create prototype database")
537
            return
538
        }
539
        defer { sqlite3_close(db) }
540
        let status = sqlite3_exec(db, "CREATE TABLE archive_samples (id TEXT PRIMARY KEY)", nil, nil, nil)
541
        XCTAssertEqual(status, SQLITE_OK)
542
    }
543

            
544
    private func makeStepCountSample() -> HKQuantitySample {
Bogdan Timofte authored 2 weeks ago
545
        makeStepCountSample(value: 42, start: 1_000)
546
    }
547

            
Bogdan Timofte authored 2 weeks ago
548
    private func makeStepCountSample(value: Double, start: TimeInterval, end: TimeInterval? = nil) -> HKQuantitySample {
Bogdan Timofte authored 2 weeks ago
549
        let quantityType = HKQuantityType.quantityType(forIdentifier: .stepCount)!
Bogdan Timofte authored 2 weeks ago
550
        let quantity = HKQuantity(unit: .count(), doubleValue: value)
551
        let startDate = Date(timeIntervalSince1970: start)
Bogdan Timofte authored 2 weeks ago
552
        let endDate = Date(timeIntervalSince1970: end ?? (start + 300))
Bogdan Timofte authored 2 weeks ago
553
        return HKQuantitySample(type: quantityType, quantity: quantity, start: startDate, end: endDate)
Bogdan Timofte authored 2 weeks ago
554
    }
555

            
Bogdan Timofte authored 2 weeks ago
556
    private func makeStepCountSamples(count: Int, startIndex: Int) -> [HKQuantitySample] {
557
        (0..<count).map { offset in
558
            let index = startIndex + offset
559
            return makeStepCountSample(
560
                value: Double((index % 97) + 1),
561
                start: 10_000 + Double(index * 600)
562
            )
563
        }
564
    }
565

            
Bogdan Timofte authored 2 weeks ago
566
    private func waitForArchiveOperation<T>(_ operation: @escaping () async throws -> T) throws -> T {
567
        let expectation = expectation(description: "archive operation")
568
        let box = AsyncResultBox<T>()
569

            
570
        Task {
571
            do {
572
                box.result = .success(try await operation())
573
            } catch {
574
                box.result = .failure(error)
575
            }
576
            expectation.fulfill()
577
        }
578

            
579
        wait(for: [expectation], timeout: 20)
580
        return try XCTUnwrap(box.result).get()
581
    }
582

            
Bogdan Timofte authored 2 weeks ago
583
    private func countRows(in tableName: String, at url: URL) throws -> Int {
584
        var db: OpaquePointer?
585
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
586
            sqlite3_close(db)
587
            XCTFail("Could not open test database")
588
            return 0
589
        }
590
        defer { sqlite3_close(db) }
591

            
592
        var statement: OpaquePointer?
593
        guard sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM \(tableName)", -1, &statement, nil) == SQLITE_OK else {
594
            sqlite3_finalize(statement)
595
            XCTFail("Could not prepare count query")
596
            return 0
597
        }
598
        defer { sqlite3_finalize(statement) }
599

            
600
        guard sqlite3_step(statement) == SQLITE_ROW else {
601
            return 0
602
        }
603
        return Int(sqlite3_column_int(statement, 0))
604
    }
605

            
Bogdan Timofte authored 2 weeks ago
606
    private func tableExists(_ tableName: String, at url: URL) throws -> Bool {
607
        var db: OpaquePointer?
608
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
609
            sqlite3_close(db)
610
            XCTFail("Could not open test database")
611
            return false
612
        }
613
        defer { sqlite3_close(db) }
614

            
615
        let sql = """
616
        SELECT 1
617
        FROM sqlite_master
618
        WHERE type = 'table' AND name = ?
619
        LIMIT 1
620
        """
621
        var statement: OpaquePointer?
622
        guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else {
623
            sqlite3_finalize(statement)
624
            XCTFail("Could not prepare table existence query")
625
            return false
626
        }
627
        defer { sqlite3_finalize(statement) }
628

            
629
        sqlite3_bind_text(statement, 1, tableName, -1, unsafeBitCast(-1, to: sqlite3_destructor_type.self))
630
        return sqlite3_step(statement) == SQLITE_ROW
631
    }
632

            
Bogdan Timofte authored 2 weeks ago
633
    private func observationIDs(at url: URL) throws -> [Int64] {
634
        var db: OpaquePointer?
635
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
636
            sqlite3_close(db)
637
            XCTFail("Could not open test database")
638
            return []
639
        }
640
        defer { sqlite3_close(db) }
641

            
642
        var statement: OpaquePointer?
643
        guard sqlite3_prepare_v2(db, "SELECT id FROM observations ORDER BY id", -1, &statement, nil) == SQLITE_OK else {
644
            sqlite3_finalize(statement)
645
            XCTFail("Could not prepare observation query")
646
            return []
647
        }
648
        defer { sqlite3_finalize(statement) }
649

            
650
        var ids: [Int64] = []
651
        while sqlite3_step(statement) == SQLITE_ROW {
652
            ids.append(sqlite3_column_int64(statement, 0))
653
        }
654
        return ids
655
    }
656

            
Bogdan Timofte authored 2 weeks ago
657
    private func sampleVersionDebugRows(at url: URL) throws -> String {
658
        var db: OpaquePointer?
659
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
660
            sqlite3_close(db)
661
            return "could not open database"
662
        }
663
        defer { sqlite3_close(db) }
664

            
665
        let sql = """
666
        SELECT v.payload_hash, v.start_date, v.end_date, v.value_kind, v.numeric_value, v.unit,
667
               v.source_revision_id, src.bundle_identifier, sr.product_type, sr.version,
668
               sr.operating_system_version, v.hk_device_id, v.metadata_id
669
        FROM sample_versions v
670
        LEFT JOIN source_revisions sr ON sr.id = v.source_revision_id
671
        LEFT JOIN sources src ON src.id = sr.source_id
672
        ORDER BY v.id
673
        """
674
        var statement: OpaquePointer?
675
        guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else {
676
            sqlite3_finalize(statement)
677
            return "could not prepare version debug query"
678
        }
679
        defer { sqlite3_finalize(statement) }
680

            
681
        var rows: [String] = []
682
        while sqlite3_step(statement) == SQLITE_ROW {
683
            rows.append((0..<13).map { index in
684
                if sqlite3_column_type(statement, Int32(index)) == SQLITE_NULL {
685
                    return "null"
686
                }
687
                if let text = sqlite3_column_text(statement, Int32(index)) {
688
                    return String(cString: text)
689
                }
690
                return "\(sqlite3_column_double(statement, Int32(index)))"
691
            }.joined(separator: "|"))
692
        }
693
        return rows.joined(separator: "\n")
694
    }
695
}