HealthProbe / HealthProbeTests / SQLiteHealthArchiveStoreTests.swift
Newer Older
646 lines | 30.792kb
Bogdan Timofte authored 2 weeks ago
1
import HealthKit
2
import SQLite3
3
import XCTest
4
@testable import HealthProbe
5

            
6
final class SQLiteHealthArchiveStoreTests: XCTestCase {
Bogdan Timofte authored 2 weeks ago
7
    private final class AsyncResultBox<T>: @unchecked Sendable {
8
        var result: Result<T, Error>?
9
    }
10

            
Bogdan Timofte authored 2 weeks ago
11
    private var temporaryDirectory: URL!
12

            
13
    override func setUpWithError() throws {
14
        temporaryDirectory = FileManager.default.temporaryDirectory
15
            .appending(path: "HealthProbeTests-\(UUID().uuidString)", directoryHint: .isDirectory)
16
        try FileManager.default.createDirectory(at: temporaryDirectory, withIntermediateDirectories: true)
17
    }
18

            
19
    override func tearDownWithError() throws {
20
        if let temporaryDirectory {
21
            try? FileManager.default.removeItem(at: temporaryDirectory)
22
        }
23
        temporaryDirectory = nil
24
    }
25

            
26
    func testFreshArchiveInitializesSchemaAndPassesIntegrity() async throws {
27
        let store = SQLiteHealthArchiveStore(databaseURL: databaseURL())
28

            
29
        let report = try await store.checkIntegrity()
30
        let records = try await store.records(for: HealthArchiveRecordRequest(
31
            sampleTypeIdentifier: HKQuantityTypeIdentifier.stepCount.rawValue
32
        ))
33

            
34
        XCTAssertTrue(report.passed)
35
        XCTAssertEqual(report.schemaVersion, 2)
36
        XCTAssertEqual(report.sqliteIntegrityStatus, "ok")
37
        XCTAssertEqual(report.foreignKeyIssueCount, 0)
38
        XCTAssertTrue(report.missingTableNames.isEmpty)
39
        XCTAssertTrue(report.requiredTableNames.contains("sample_visibility_ranges"))
40
        XCTAssertTrue(report.requiredTableNames.contains("daily_type_aggregates"))
41
        XCTAssertTrue(records.isEmpty)
42
    }
43

            
44
    func testPrototypeArchiveIsResetAndReinitializedAsV2() async throws {
45
        let url = databaseURL()
46
        try createPrototypeDatabase(at: url)
47
        let store = SQLiteHealthArchiveStore(databaseURL: url)
48

            
49
        let report = try await store.checkIntegrity()
50

            
51
        XCTAssertTrue(report.passed)
52
        XCTAssertEqual(report.schemaVersion, 2)
53
        XCTAssertTrue(report.missingTableNames.isEmpty)
54
    }
55

            
56
    func testRepeatedSamplePageDoesNotDuplicateIdentityOrVersion() async throws {
57
        let url = databaseURL()
58
        let store = SQLiteHealthArchiveStore(databaseURL: url)
59
        let sample = makeStepCountSample()
60

            
61
        let firstWrite = try await store.upsertSamples([sample], observedAt: Date(timeIntervalSince1970: 2_000))
62
        let secondWrite = try await store.upsertSamples([sample], observedAt: Date(timeIntervalSince1970: 2_060))
63
        let records = try await store.records(for: HealthArchiveRecordRequest(
64
            sampleTypeIdentifier: sample.sampleType.identifier
65
        ))
66
        let report = try await store.checkIntegrity()
67
        let versionDebugRows = try sampleVersionDebugRows(at: url)
68

            
69
        XCTAssertEqual(firstWrite.insertedCount, 1)
70
        XCTAssertEqual(firstWrite.updatedCount, 0)
71
        XCTAssertEqual(firstWrite.unchangedCount, 0)
72
        XCTAssertEqual(try countRows(in: "samples", at: url), 1)
73
        XCTAssertEqual(try countRows(in: "sample_versions", at: url), 1, versionDebugRows)
74
        XCTAssertEqual(try countRows(in: "sample_visibility_ranges", at: url), 1)
75
        XCTAssertEqual(try countRows(in: "source_revisions", at: url), 1)
Bogdan Timofte authored 2 weeks ago
76
        XCTAssertFalse(try tableExists("archive_samples", at: url))
Bogdan Timofte authored 2 weeks ago
77
        XCTAssertEqual(secondWrite.insertedCount, 0)
78
        XCTAssertEqual(secondWrite.updatedCount, 0)
79
        XCTAssertEqual(secondWrite.unchangedCount, 1)
80
        XCTAssertEqual(records.count, 1)
81
        XCTAssertEqual(records.first?.displayValue, "42.0 count")
82
        XCTAssertTrue(report.passed)
83
    }
84

            
Bogdan Timofte authored 2 weeks ago
85
    func testVerificationUsesArchiveV2TablesWithoutLegacyMirror() async throws {
86
        let url = databaseURL()
87
        let store = SQLiteHealthArchiveStore(databaseURL: url)
88
        let sample = makeStepCountSample()
89

            
90
        _ = try await store.upsertSamples([sample], observedAt: Date(timeIntervalSince1970: 2_000))
91
        try await store.markVerification(sampleType: sample.sampleType, verifiedAt: Date(timeIntervalSince1970: 2_060))
92
        let observationIDs = try observationIDs(at: url)
93

            
94
        XCTAssertEqual(observationIDs.count, 2)
Bogdan Timofte authored 2 weeks ago
95
        XCTAssertEqual(try countRows(in: "observation_type_runs", at: url), 2)
96
        XCTAssertEqual(try countRows(in: "observation_type_runs WHERE observation_id = \(observationIDs[0]) AND inserted_event_count = 1", at: url), 1)
Bogdan Timofte authored 2 weeks ago
97
        XCTAssertEqual(try countRows(in: "observation_type_summaries WHERE observation_id = \(observationIDs[1]) AND visible_record_count = 1", at: url), 1)
98
        XCTAssertFalse(try tableExists("archive_samples", at: url))
99
    }
100

            
Bogdan Timofte authored 2 weeks ago
101
    func testDiffSummaryAndRecordsBetweenObservationsUseSQLVisibility() async throws {
102
        let url = databaseURL()
103
        let store = SQLiteHealthArchiveStore(databaseURL: url)
104
        let firstSample = makeStepCountSample(value: 42, start: 1_000)
105
        let secondSample = makeStepCountSample(value: 7, start: 2_000)
106
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
107

            
108
        _ = try await store.upsertSamples([firstSample], observedAt: Date(timeIntervalSince1970: 3_000))
109
        _ = try await store.upsertSamples([firstSample, secondSample], observedAt: Date(timeIntervalSince1970: 3_060))
110
        try await store.recordDisappearance(
111
            sampleUUIDHash: HashService.sampleUUIDHash(firstSample.uuid.uuidString),
112
            sampleTypeIdentifier: typeIdentifier,
113
            observedMissingAt: Date(timeIntervalSince1970: 3_120)
114
        )
115
        let observationIDs = try observationIDs(at: url)
116
        XCTAssertEqual(observationIDs.count, 3)
117

            
118
        let appearedSummary = try await store.diffSummary(HealthArchiveDiffRequest(
119
            fromObservationID: observationIDs[0],
120
            toObservationID: observationIDs[1],
121
            sampleTypeIdentifier: typeIdentifier
122
        ))
123
        let appearedRecords = try await store.diffRecords(HealthArchiveDiffRecordRequest(
124
            fromObservationID: observationIDs[0],
125
            toObservationID: observationIDs[1],
126
            sampleTypeIdentifier: typeIdentifier,
127
            kind: .appeared,
128
            limit: 10
129
        ))
130
        let disappearedSummary = try await store.diffSummary(HealthArchiveDiffRequest(
131
            fromObservationID: observationIDs[1],
132
            toObservationID: observationIDs[2],
133
            sampleTypeIdentifier: typeIdentifier
134
        ))
135
        let disappearedRecords = try await store.diffRecords(HealthArchiveDiffRecordRequest(
136
            fromObservationID: observationIDs[1],
137
            toObservationID: observationIDs[2],
138
            sampleTypeIdentifier: typeIdentifier,
139
            kind: .disappeared,
140
            limit: 10
141
        ))
142

            
143
        XCTAssertEqual(appearedSummary.appearedCount, 1)
144
        XCTAssertEqual(appearedSummary.disappearedCount, 0)
145
        XCTAssertEqual(appearedSummary.representationChangedCount, 0)
146
        XCTAssertEqual(appearedRecords.map(\.displayValue), ["7.0 count"])
147
        XCTAssertEqual(disappearedSummary.appearedCount, 0)
148
        XCTAssertEqual(disappearedSummary.disappearedCount, 1)
149
        XCTAssertEqual(disappearedSummary.representationChangedCount, 0)
150
        XCTAssertEqual(disappearedRecords.map(\.displayValue), ["42.0 count"])
151
        XCTAssertNotNil(disappearedRecords.first?.disappearedAt)
Bogdan Timofte authored 2 weeks ago
152

            
153
        let exportRequest = HealthArchiveReportRequest(
154
            reportID: UUID(),
155
            title: "Disappeared Step Count Export",
156
            typeIdentifierFilter: typeIdentifier,
157
            diffFromObservationID: observationIDs[1],
158
            diffToObservationID: observationIDs[2],
159
            diffKind: .disappeared
160
        )
161
        let exportPreview = try await store.exportPreview(exportRequest)
162
        let exportURL = try await store.exportReport(exportRequest)
163

            
164
        XCTAssertEqual(exportPreview.estimatedRecordCount, 1)
165
        XCTAssertTrue(FileManager.default.fileExists(atPath: exportURL.path))
166
        XCTAssertEqual(try countRows(in: "export_manifests WHERE record_count = 1 AND from_observation_id = \(observationIDs[1]) AND to_observation_id = \(observationIDs[2])", at: url), 1)
167
    }
168

            
169
    func testExportPreviewAndReportUseSQLiteManifest() async throws {
170
        let url = databaseURL()
171
        let store = SQLiteHealthArchiveStore(databaseURL: url)
172
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
173
        let samples = [
174
            makeStepCountSample(value: 42, start: 1_000),
175
            makeStepCountSample(value: 7, start: 2_000)
176
        ]
177

            
178
        _ = try await store.upsertSamples(samples, observedAt: Date(timeIntervalSince1970: 3_000))
179

            
180
        let request = HealthArchiveReportRequest(
181
            reportID: UUID(),
182
            title: "Step Count Export",
183
            typeIdentifierFilter: typeIdentifier
184
        )
185
        let preview = try await store.exportPreview(request)
186
        let exportURL = try await store.exportReport(request)
187

            
188
        XCTAssertEqual(preview.estimatedRecordCount, 2)
189
        XCTAssertTrue(FileManager.default.fileExists(atPath: exportURL.path))
190
        XCTAssertEqual(try countRows(in: "export_manifests", at: url), 1)
191
        XCTAssertEqual(try countRows(in: "export_manifests WHERE record_count = 2", at: url), 1)
192

            
193
        let cache = try CoreDataArchiveCacheStore(inMemory: true)
194
        let rebuild = try cache.rebuild(fromArchiveAt: url)
195
        XCTAssertEqual(rebuild.exportManifestRows, 1)
Bogdan Timofte authored 2 weeks ago
196
    }
197

            
Bogdan Timofte authored 2 weeks ago
198
    func testGroupedObservationKeepsPageWritesDeletesAndVerificationTogether() async throws {
199
        let url = databaseURL()
200
        let store = SQLiteHealthArchiveStore(databaseURL: url)
201
        let firstSample = makeStepCountSample(value: 42, start: 1_000)
202
        let secondSample = makeStepCountSample(value: 7, start: 2_000)
203
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
204

            
205
        _ = try await store.upsertSamples([firstSample], observedAt: Date(timeIntervalSince1970: 3_000))
206
        let observationID = try await store.beginObservation(
207
            observedAt: Date(timeIntervalSince1970: 3_060),
208
            triggerReason: "manual",
209
            selectedTypeSetHash: "selected-types"
210
        )
211
        _ = try await store.upsertSamples(
212
            [secondSample],
213
            observedAt: Date(timeIntervalSince1970: 3_060),
214
            observationID: observationID
215
        )
216
        try await store.recordDisappearance(
217
            sampleUUIDHash: HashService.sampleUUIDHash(firstSample.uuid.uuidString),
218
            sampleTypeIdentifier: typeIdentifier,
219
            observedMissingAt: Date(timeIntervalSince1970: 3_060),
220
            observationID: observationID
221
        )
222
        try await store.markVerification(
223
            sampleType: secondSample.sampleType,
224
            verifiedAt: Date(timeIntervalSince1970: 3_060),
225
            observationID: observationID
226
        )
227
        try await store.finishObservation(
228
            observationID: observationID,
229
            status: "completed",
230
            endedAt: Date(timeIntervalSince1970: 3_070)
231
        )
232

            
233
        let observationIDs = try observationIDs(at: url)
234
        let summary = try await store.diffSummary(HealthArchiveDiffRequest(
235
            fromObservationID: observationIDs[0],
236
            toObservationID: observationID,
237
            sampleTypeIdentifier: typeIdentifier
238
        ))
239

            
240
        XCTAssertEqual(observationIDs.count, 2)
241
        XCTAssertEqual(summary.appearedCount, 1)
242
        XCTAssertEqual(summary.disappearedCount, 1)
243
        XCTAssertEqual(try countRows(in: "observations WHERE id = \(observationID) AND status = 'completed' AND selected_type_set_hash = 'selected-types'", at: url), 1)
244
        XCTAssertEqual(try countRows(in: "sample_observation_events WHERE observation_id = \(observationID)", at: url), 2)
245
        XCTAssertEqual(try countRows(in: "observation_type_runs WHERE observation_id = \(observationID) AND inserted_event_count = 1 AND deleted_event_count = 1 AND verified_visible_count = 1 AND status = 'completed'", at: url), 1)
246
    }
247

            
Bogdan Timofte authored 2 weeks ago
248
    func testLargeSyntheticDiffReturnsCountsAndBoundedPages() async throws {
249
        let url = databaseURL()
250
        let store = SQLiteHealthArchiveStore(databaseURL: url)
251
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
252
        let initialCount = 1_200
253
        let appearedCount = 180
254
        let disappearedCount = 160
255
        let pageSize = 25
256
        let initialSamples = makeStepCountSamples(count: initialCount, startIndex: 0)
257
        let appearedSamples = makeStepCountSamples(count: appearedCount, startIndex: initialCount)
258

            
259
        _ = try await store.upsertSamples(initialSamples, observedAt: Date(timeIntervalSince1970: 1_000_000))
260
        _ = try await store.upsertSamples(initialSamples + appearedSamples, observedAt: Date(timeIntervalSince1970: 1_000_060))
261
        for (offset, sample) in initialSamples.prefix(disappearedCount).enumerated() {
262
            try await store.recordDisappearance(
263
                sampleUUIDHash: HashService.sampleUUIDHash(sample.uuid.uuidString),
264
                sampleTypeIdentifier: typeIdentifier,
265
                observedMissingAt: Date(timeIntervalSince1970: 1_000_120 + Double(offset))
266
            )
267
        }
268

            
269
        let observationIDs = try observationIDs(at: url)
270
        let firstObservationID = try XCTUnwrap(observationIDs.first)
271
        let lastObservationID = try XCTUnwrap(observationIDs.last)
272
        let queryStartedAt = Date()
273
        let summary = try await store.diffSummary(HealthArchiveDiffRequest(
274
            fromObservationID: firstObservationID,
275
            toObservationID: lastObservationID,
276
            sampleTypeIdentifier: typeIdentifier
277
        ))
278
        let firstPage = try await store.diffRecords(HealthArchiveDiffRecordRequest(
279
            fromObservationID: firstObservationID,
280
            toObservationID: lastObservationID,
281
            sampleTypeIdentifier: typeIdentifier,
282
            kind: .appeared,
283
            limit: pageSize
284
        ))
285
        let firstPageLastRecord = try XCTUnwrap(firstPage.last)
286
        let secondPage = try await store.diffRecords(HealthArchiveDiffRecordRequest(
287
            fromObservationID: firstObservationID,
288
            toObservationID: lastObservationID,
289
            sampleTypeIdentifier: typeIdentifier,
290
            kind: .appeared,
291
            afterCursor: RecordCursor(
292
                startDate: firstPageLastRecord.startDate,
293
                strictFingerprint: firstPageLastRecord.strictFingerprint
294
            ),
295
            limit: pageSize
296
        ))
297
        let queryElapsedSeconds = Date().timeIntervalSince(queryStartedAt)
298

            
299
        XCTAssertEqual(summary.appearedCount, appearedCount)
300
        XCTAssertEqual(summary.disappearedCount, disappearedCount)
301
        XCTAssertEqual(summary.representationChangedCount, 0)
302
        XCTAssertEqual(firstPage.count, pageSize)
303
        XCTAssertEqual(secondPage.count, pageSize)
304
        XCTAssertTrue(Set(firstPage.map(\.id)).isDisjoint(with: Set(secondPage.map(\.id))))
305
        XCTAssertLessThan(queryElapsedSeconds, 10)
306
    }
307

            
Bogdan Timofte authored 2 weeks ago
308
    func testLargeSyntheticDiffQueryMetrics() throws {
309
        let url = databaseURL()
310
        let store = SQLiteHealthArchiveStore(databaseURL: url)
311
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
312
        let initialCount = 900
313
        let appearedCount = 120
314
        let disappearedCount = 100
315
        let pageSize = 25
316
        let initialSamples = makeStepCountSamples(count: initialCount, startIndex: 0)
317
        let appearedSamples = makeStepCountSamples(count: appearedCount, startIndex: initialCount)
318

            
319
        try waitForArchiveOperation {
320
            _ = try await store.upsertSamples(initialSamples, observedAt: Date(timeIntervalSince1970: 1_200_000))
321
            _ = try await store.upsertSamples(initialSamples + appearedSamples, observedAt: Date(timeIntervalSince1970: 1_200_060))
322
            for (offset, sample) in initialSamples.prefix(disappearedCount).enumerated() {
323
                try await store.recordDisappearance(
324
                    sampleUUIDHash: HashService.sampleUUIDHash(sample.uuid.uuidString),
325
                    sampleTypeIdentifier: typeIdentifier,
326
                    observedMissingAt: Date(timeIntervalSince1970: 1_200_120 + Double(offset))
327
                )
328
            }
329
        }
330

            
331
        let observationIDs = try observationIDs(at: url)
332
        let firstObservationID = try XCTUnwrap(observationIDs.first)
333
        let lastObservationID = try XCTUnwrap(observationIDs.last)
334
        let options = XCTMeasureOptions()
335
        options.iterationCount = 3
336

            
337
        measure(metrics: [XCTClockMetric(), XCTMemoryMetric()], options: options) {
338
            do {
339
                let result = try waitForArchiveOperation {
340
                    let summary = try await store.diffSummary(HealthArchiveDiffRequest(
341
                        fromObservationID: firstObservationID,
342
                        toObservationID: lastObservationID,
343
                        sampleTypeIdentifier: typeIdentifier
344
                    ))
345
                    let records = try await store.diffRecords(HealthArchiveDiffRecordRequest(
346
                        fromObservationID: firstObservationID,
347
                        toObservationID: lastObservationID,
348
                        sampleTypeIdentifier: typeIdentifier,
349
                        kind: .appeared,
350
                        limit: pageSize
351
                    ))
352
                    return (summary, records)
353
                }
354
                XCTAssertEqual(result.0.appearedCount, appearedCount)
355
                XCTAssertEqual(result.0.disappearedCount, disappearedCount)
356
                XCTAssertEqual(result.1.count, pageSize)
357
            } catch {
358
                XCTFail("Measured archive query failed: \(error)")
359
            }
360
        }
361
    }
362

            
Bogdan Timofte authored 2 weeks ago
363
    func testAggregateComparisonUsesMaterializedDailyAggregates() async throws {
364
        let url = databaseURL()
365
        let store = SQLiteHealthArchiveStore(databaseURL: url)
366
        let firstSample = makeStepCountSample(value: 42, start: 1_000)
367
        let secondSample = makeStepCountSample(value: 7, start: 2_000)
368
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
369

            
370
        _ = try await store.upsertSamples([firstSample], observedAt: Date(timeIntervalSince1970: 3_000))
371
        _ = try await store.upsertSamples([firstSample, secondSample], observedAt: Date(timeIntervalSince1970: 3_060))
372
        let observationIDs = try observationIDs(at: url)
373
        XCTAssertEqual(observationIDs.count, 2)
374

            
375
        let rows = try await store.aggregateComparison(HealthArchiveAggregateComparisonRequest(
376
            fromObservationID: observationIDs[0],
377
            toObservationID: observationIDs[1],
378
            sampleTypeIdentifier: typeIdentifier,
379
            limit: 10
380
        ))
381

            
382
        XCTAssertEqual(rows.count, 1)
383
        XCTAssertEqual(rows.first?.sampleTypeIdentifier, typeIdentifier)
384
        XCTAssertEqual(rows.first?.fromVisibleRecordCount, 1)
385
        XCTAssertEqual(rows.first?.toVisibleRecordCount, 2)
386
        XCTAssertEqual(rows.first?.visibleRecordDelta, 1)
387
        XCTAssertEqual(rows.first?.fromValueSum, 42)
388
        XCTAssertEqual(rows.first?.toValueSum, 49)
389
        XCTAssertEqual(rows.first?.valueSumDelta, 7)
390
    }
391

            
Bogdan Timofte authored 2 weeks ago
392
    func testSourceProvenanceBreakdownReturnsVisibleSourceRows() async throws {
393
        let url = databaseURL()
394
        let store = SQLiteHealthArchiveStore(databaseURL: url)
395
        let sample = makeStepCountSample(value: 42, start: 1_000)
396
        let typeIdentifier = sample.sampleType.identifier
397

            
398
        _ = try await store.upsertSamples([sample], observedAt: Date(timeIntervalSince1970: 3_000))
399

            
400
        let rows = try await store.sourceProvenanceBreakdown(HealthArchiveSourceProvenanceRequest(
401
            visibleAtObservationID: nil,
402
            sampleTypeIdentifier: typeIdentifier,
403
            limit: 10
404
        ))
405

            
406
        XCTAssertEqual(rows.count, 1)
407
        XCTAssertEqual(rows.first?.sampleTypeIdentifier, typeIdentifier)
408
        XCTAssertEqual(rows.first?.visibleRecordCount, 1)
409
        XCTAssertEqual(rows.first?.valueSum, 42)
410
        XCTAssertEqual(rows.first?.sourceBundleIdentifier, sample.sourceRevision.source.bundleIdentifier)
411
    }
412

            
413
    func testConsolidationEvidenceClassifiesStableCountDropAsConsolidationLikely() async throws {
414
        let url = databaseURL()
415
        let store = SQLiteHealthArchiveStore(databaseURL: url)
416
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
417

            
418
        let firstOldSample = makeStepCountSample(value: 10, start: 1_000, end: 1_500)
419
        let secondOldSample = makeStepCountSample(value: 20, start: 1_600, end: 2_000)
420
        let consolidatedSample = makeStepCountSample(value: 30, start: 1_000, end: 2_500)
421

            
422
        _ = try await store.upsertSamples([firstOldSample, secondOldSample], observedAt: Date(timeIntervalSince1970: 3_000))
423
        _ = try await store.upsertSamples([consolidatedSample], observedAt: Date(timeIntervalSince1970: 3_060))
424
        try await store.recordDisappearance(sampleUUIDHash: HashService.sampleUUIDHash(firstOldSample.uuid.uuidString), sampleTypeIdentifier: typeIdentifier, observedMissingAt: Date(timeIntervalSince1970: 3_120))
425
        try await store.recordDisappearance(sampleUUIDHash: HashService.sampleUUIDHash(secondOldSample.uuid.uuidString), sampleTypeIdentifier: typeIdentifier, observedMissingAt: Date(timeIntervalSince1970: 3_180))
426

            
427
        let observationIDs = try observationIDs(at: url)
428
        XCTAssertEqual(observationIDs.count, 4)
429

            
430
        let rows = try await store.consolidationEvidence(HealthArchiveConsolidationEvidenceRequest(
431
            fromObservationID: observationIDs[0],
432
            toObservationID: observationIDs[observationIDs.count - 1],
433
            sampleTypeIdentifier: typeIdentifier
434
        ))
435

            
436
        XCTAssertEqual(rows.count, 1)
437
        XCTAssertEqual(rows.first?.sampleTypeIdentifier, typeIdentifier)
438
        XCTAssertEqual(rows.first?.disappearedCount, 2)
439
        XCTAssertEqual(rows.first?.appearedCount, 1)
440
        XCTAssertEqual(rows.first?.fromVisibleRecordCount, 2)
441
        XCTAssertEqual(rows.first?.toVisibleRecordCount, 1)
442
        XCTAssertEqual(rows.first?.fromValueSum, 30)
443
        XCTAssertEqual(rows.first?.toValueSum, 30)
444
        XCTAssertEqual(rows.first?.label, "consolidation_likely")
445
        XCTAssertTrue(rows.first?.sourceCompatible == true)
446
    }
447

            
448
    func testConsolidationEvidenceClassifiesDenseStableShiftAsAggregateChanged() async throws {
449
        let url = databaseURL()
450
        let store = SQLiteHealthArchiveStore(databaseURL: url)
451
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
452

            
453
        let firstOldSample = makeStepCountSample(value: 10, start: 1_000, end: 1_500)
454
        let secondOldSample = makeStepCountSample(value: 20, start: 5_000, end: 6_000)
455
        let denseSample = makeStepCountSample(value: 30, start: 1_000, end: 1_200)
456

            
457
        _ = try await store.upsertSamples([firstOldSample, secondOldSample], observedAt: Date(timeIntervalSince1970: 3_000))
458
        _ = try await store.upsertSamples([denseSample], observedAt: Date(timeIntervalSince1970: 3_060))
459
        try await store.recordDisappearance(sampleUUIDHash: HashService.sampleUUIDHash(firstOldSample.uuid.uuidString), sampleTypeIdentifier: typeIdentifier, observedMissingAt: Date(timeIntervalSince1970: 3_120))
460
        try await store.recordDisappearance(sampleUUIDHash: HashService.sampleUUIDHash(secondOldSample.uuid.uuidString), sampleTypeIdentifier: typeIdentifier, observedMissingAt: Date(timeIntervalSince1970: 3_180))
461

            
462
        let observationIDs = try observationIDs(at: url)
463
        XCTAssertEqual(observationIDs.count, 4)
464

            
465
        let rows = try await store.consolidationEvidence(HealthArchiveConsolidationEvidenceRequest(
466
            fromObservationID: observationIDs[0],
467
            toObservationID: observationIDs[observationIDs.count - 1],
468
            sampleTypeIdentifier: typeIdentifier
469
        ))
470

            
471
        XCTAssertEqual(rows.count, 1)
472
        XCTAssertEqual(rows.first?.label, "aggregate_changed")
473
        XCTAssertEqual(rows.first?.fromVisibleRecordCount, 2)
474
        XCTAssertEqual(rows.first?.toVisibleRecordCount, 1)
475
        XCTAssertEqual(rows.first?.fromValueSum, 30)
476
        XCTAssertEqual(rows.first?.toValueSum, 30)
477
    }
478

            
Bogdan Timofte authored 2 weeks ago
479
    private func databaseURL() -> URL {
480
        temporaryDirectory.appending(path: "Archive.sqlite")
481
    }
482

            
483
    private func createPrototypeDatabase(at url: URL) throws {
484
        var db: OpaquePointer?
485
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
486
            sqlite3_close(db)
487
            XCTFail("Could not create prototype database")
488
            return
489
        }
490
        defer { sqlite3_close(db) }
491
        let status = sqlite3_exec(db, "CREATE TABLE archive_samples (id TEXT PRIMARY KEY)", nil, nil, nil)
492
        XCTAssertEqual(status, SQLITE_OK)
493
    }
494

            
495
    private func makeStepCountSample() -> HKQuantitySample {
Bogdan Timofte authored 2 weeks ago
496
        makeStepCountSample(value: 42, start: 1_000)
497
    }
498

            
Bogdan Timofte authored 2 weeks ago
499
    private func makeStepCountSample(value: Double, start: TimeInterval, end: TimeInterval? = nil) -> HKQuantitySample {
Bogdan Timofte authored 2 weeks ago
500
        let quantityType = HKQuantityType.quantityType(forIdentifier: .stepCount)!
Bogdan Timofte authored 2 weeks ago
501
        let quantity = HKQuantity(unit: .count(), doubleValue: value)
502
        let startDate = Date(timeIntervalSince1970: start)
Bogdan Timofte authored 2 weeks ago
503
        let endDate = Date(timeIntervalSince1970: end ?? (start + 300))
Bogdan Timofte authored 2 weeks ago
504
        return HKQuantitySample(type: quantityType, quantity: quantity, start: startDate, end: endDate)
Bogdan Timofte authored 2 weeks ago
505
    }
506

            
Bogdan Timofte authored 2 weeks ago
507
    private func makeStepCountSamples(count: Int, startIndex: Int) -> [HKQuantitySample] {
508
        (0..<count).map { offset in
509
            let index = startIndex + offset
510
            return makeStepCountSample(
511
                value: Double((index % 97) + 1),
512
                start: 10_000 + Double(index * 600)
513
            )
514
        }
515
    }
516

            
Bogdan Timofte authored 2 weeks ago
517
    private func waitForArchiveOperation<T>(_ operation: @escaping () async throws -> T) throws -> T {
518
        let expectation = expectation(description: "archive operation")
519
        let box = AsyncResultBox<T>()
520

            
521
        Task {
522
            do {
523
                box.result = .success(try await operation())
524
            } catch {
525
                box.result = .failure(error)
526
            }
527
            expectation.fulfill()
528
        }
529

            
530
        wait(for: [expectation], timeout: 20)
531
        return try XCTUnwrap(box.result).get()
532
    }
533

            
Bogdan Timofte authored 2 weeks ago
534
    private func countRows(in tableName: String, at url: URL) throws -> Int {
535
        var db: OpaquePointer?
536
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
537
            sqlite3_close(db)
538
            XCTFail("Could not open test database")
539
            return 0
540
        }
541
        defer { sqlite3_close(db) }
542

            
543
        var statement: OpaquePointer?
544
        guard sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM \(tableName)", -1, &statement, nil) == SQLITE_OK else {
545
            sqlite3_finalize(statement)
546
            XCTFail("Could not prepare count query")
547
            return 0
548
        }
549
        defer { sqlite3_finalize(statement) }
550

            
551
        guard sqlite3_step(statement) == SQLITE_ROW else {
552
            return 0
553
        }
554
        return Int(sqlite3_column_int(statement, 0))
555
    }
556

            
Bogdan Timofte authored 2 weeks ago
557
    private func tableExists(_ tableName: String, at url: URL) throws -> Bool {
558
        var db: OpaquePointer?
559
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
560
            sqlite3_close(db)
561
            XCTFail("Could not open test database")
562
            return false
563
        }
564
        defer { sqlite3_close(db) }
565

            
566
        let sql = """
567
        SELECT 1
568
        FROM sqlite_master
569
        WHERE type = 'table' AND name = ?
570
        LIMIT 1
571
        """
572
        var statement: OpaquePointer?
573
        guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else {
574
            sqlite3_finalize(statement)
575
            XCTFail("Could not prepare table existence query")
576
            return false
577
        }
578
        defer { sqlite3_finalize(statement) }
579

            
580
        sqlite3_bind_text(statement, 1, tableName, -1, unsafeBitCast(-1, to: sqlite3_destructor_type.self))
581
        return sqlite3_step(statement) == SQLITE_ROW
582
    }
583

            
Bogdan Timofte authored 2 weeks ago
584
    private func observationIDs(at url: URL) throws -> [Int64] {
585
        var db: OpaquePointer?
586
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
587
            sqlite3_close(db)
588
            XCTFail("Could not open test database")
589
            return []
590
        }
591
        defer { sqlite3_close(db) }
592

            
593
        var statement: OpaquePointer?
594
        guard sqlite3_prepare_v2(db, "SELECT id FROM observations ORDER BY id", -1, &statement, nil) == SQLITE_OK else {
595
            sqlite3_finalize(statement)
596
            XCTFail("Could not prepare observation query")
597
            return []
598
        }
599
        defer { sqlite3_finalize(statement) }
600

            
601
        var ids: [Int64] = []
602
        while sqlite3_step(statement) == SQLITE_ROW {
603
            ids.append(sqlite3_column_int64(statement, 0))
604
        }
605
        return ids
606
    }
607

            
Bogdan Timofte authored 2 weeks ago
608
    private func sampleVersionDebugRows(at url: URL) throws -> String {
609
        var db: OpaquePointer?
610
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
611
            sqlite3_close(db)
612
            return "could not open database"
613
        }
614
        defer { sqlite3_close(db) }
615

            
616
        let sql = """
617
        SELECT v.payload_hash, v.start_date, v.end_date, v.value_kind, v.numeric_value, v.unit,
618
               v.source_revision_id, src.bundle_identifier, sr.product_type, sr.version,
619
               sr.operating_system_version, v.hk_device_id, v.metadata_id
620
        FROM sample_versions v
621
        LEFT JOIN source_revisions sr ON sr.id = v.source_revision_id
622
        LEFT JOIN sources src ON src.id = sr.source_id
623
        ORDER BY v.id
624
        """
625
        var statement: OpaquePointer?
626
        guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else {
627
            sqlite3_finalize(statement)
628
            return "could not prepare version debug query"
629
        }
630
        defer { sqlite3_finalize(statement) }
631

            
632
        var rows: [String] = []
633
        while sqlite3_step(statement) == SQLITE_ROW {
634
            rows.append((0..<13).map { index in
635
                if sqlite3_column_type(statement, Int32(index)) == SQLITE_NULL {
636
                    return "null"
637
                }
638
                if let text = sqlite3_column_text(statement, Int32(index)) {
639
                    return String(cString: text)
640
                }
641
                return "\(sqlite3_column_double(statement, Int32(index)))"
642
            }.joined(separator: "|"))
643
        }
644
        return rows.joined(separator: "\n")
645
    }
646
}