HealthProbe / HealthProbeTests / SQLiteHealthArchiveStoreTests.swift
Newer Older
765 lines | 37.014kb
Bogdan Timofte authored 2 weeks ago
1
import HealthKit
2
import SQLite3
3
import XCTest
4
@testable import HealthProbe
5

            
6
final class SQLiteHealthArchiveStoreTests: XCTestCase {
Bogdan Timofte authored 2 weeks ago
7
    private final class AsyncResultBox<T>: @unchecked Sendable {
8
        var result: Result<T, Error>?
9
    }
10

            
Bogdan Timofte authored 2 weeks ago
11
    private var temporaryDirectory: URL!
12

            
13
    override func setUpWithError() throws {
14
        temporaryDirectory = FileManager.default.temporaryDirectory
15
            .appending(path: "HealthProbeTests-\(UUID().uuidString)", directoryHint: .isDirectory)
16
        try FileManager.default.createDirectory(at: temporaryDirectory, withIntermediateDirectories: true)
17
    }
18

            
19
    override func tearDownWithError() throws {
20
        if let temporaryDirectory {
21
            try? FileManager.default.removeItem(at: temporaryDirectory)
22
        }
23
        temporaryDirectory = nil
24
    }
25

            
26
    func testFreshArchiveInitializesSchemaAndPassesIntegrity() async throws {
27
        let store = SQLiteHealthArchiveStore(databaseURL: databaseURL())
28

            
29
        let report = try await store.checkIntegrity()
30
        let records = try await store.records(for: HealthArchiveRecordRequest(
31
            sampleTypeIdentifier: HKQuantityTypeIdentifier.stepCount.rawValue
32
        ))
33

            
34
        XCTAssertTrue(report.passed)
35
        XCTAssertEqual(report.schemaVersion, 2)
36
        XCTAssertEqual(report.sqliteIntegrityStatus, "ok")
37
        XCTAssertEqual(report.foreignKeyIssueCount, 0)
38
        XCTAssertTrue(report.missingTableNames.isEmpty)
39
        XCTAssertTrue(report.requiredTableNames.contains("sample_visibility_ranges"))
40
        XCTAssertTrue(report.requiredTableNames.contains("daily_type_aggregates"))
41
        XCTAssertTrue(records.isEmpty)
42
    }
43

            
44
    func testPrototypeArchiveIsResetAndReinitializedAsV2() async throws {
45
        let url = databaseURL()
46
        try createPrototypeDatabase(at: url)
47
        let store = SQLiteHealthArchiveStore(databaseURL: url)
48

            
49
        let report = try await store.checkIntegrity()
50

            
51
        XCTAssertTrue(report.passed)
52
        XCTAssertEqual(report.schemaVersion, 2)
53
        XCTAssertTrue(report.missingTableNames.isEmpty)
54
    }
55

            
56
    func testRepeatedSamplePageDoesNotDuplicateIdentityOrVersion() async throws {
57
        let url = databaseURL()
58
        let store = SQLiteHealthArchiveStore(databaseURL: url)
59
        let sample = makeStepCountSample()
60

            
61
        let firstWrite = try await store.upsertSamples([sample], observedAt: Date(timeIntervalSince1970: 2_000))
62
        let secondWrite = try await store.upsertSamples([sample], observedAt: Date(timeIntervalSince1970: 2_060))
63
        let records = try await store.records(for: HealthArchiveRecordRequest(
64
            sampleTypeIdentifier: sample.sampleType.identifier
65
        ))
66
        let report = try await store.checkIntegrity()
67
        let versionDebugRows = try sampleVersionDebugRows(at: url)
Bogdan Timofte authored 6 days ago
68
        let visibilityDebugRows = try visibilityRangeDebugRows(at: url)
Bogdan Timofte authored 2 weeks ago
69

            
70
        XCTAssertEqual(firstWrite.insertedCount, 1)
71
        XCTAssertEqual(firstWrite.updatedCount, 0)
72
        XCTAssertEqual(firstWrite.unchangedCount, 0)
73
        XCTAssertEqual(try countRows(in: "samples", at: url), 1)
74
        XCTAssertEqual(try countRows(in: "sample_versions", at: url), 1, versionDebugRows)
Bogdan Timofte authored 5 days ago
75
        XCTAssertEqual(try countRows(in: "sample_observation_events", at: url), 1)
76
        XCTAssertEqual(try countRows(in: "sample_observation_events WHERE event_kind = 'appeared'", at: url), 1)
77
        XCTAssertEqual(try countRows(in: "sample_observation_events WHERE event_kind = 'verified'", at: url), 0)
Bogdan Timofte authored 6 days ago
78
        XCTAssertEqual(try countRows(in: "sample_visibility_ranges", at: url), 1, visibilityDebugRows)
Bogdan Timofte authored 2 weeks ago
79
        XCTAssertEqual(try countRows(in: "source_revisions", at: url), 1)
Bogdan Timofte authored 2 weeks ago
80
        XCTAssertFalse(try tableExists("archive_samples", at: url))
Bogdan Timofte authored 2 weeks ago
81
        XCTAssertEqual(secondWrite.insertedCount, 0)
82
        XCTAssertEqual(secondWrite.updatedCount, 0)
83
        XCTAssertEqual(secondWrite.unchangedCount, 1)
84
        XCTAssertEqual(records.count, 1)
85
        XCTAssertEqual(records.first?.displayValue, "42.0 count")
86
        XCTAssertTrue(report.passed)
87
    }
88

            
Bogdan Timofte authored 2 weeks ago
89
    func testVerificationUsesArchiveV2TablesWithoutLegacyMirror() async throws {
90
        let url = databaseURL()
91
        let store = SQLiteHealthArchiveStore(databaseURL: url)
92
        let sample = makeStepCountSample()
93

            
94
        _ = try await store.upsertSamples([sample], observedAt: Date(timeIntervalSince1970: 2_000))
95
        try await store.markVerification(sampleType: sample.sampleType, verifiedAt: Date(timeIntervalSince1970: 2_060))
96
        let observationIDs = try observationIDs(at: url)
97

            
98
        XCTAssertEqual(observationIDs.count, 2)
Bogdan Timofte authored 2 weeks ago
99
        XCTAssertEqual(try countRows(in: "observation_type_runs", at: url), 2)
100
        XCTAssertEqual(try countRows(in: "observation_type_runs WHERE observation_id = \(observationIDs[0]) AND inserted_event_count = 1", at: url), 1)
Bogdan Timofte authored 2 weeks ago
101
        XCTAssertEqual(try countRows(in: "observation_type_summaries WHERE observation_id = \(observationIDs[1]) AND visible_record_count = 1", at: url), 1)
Bogdan Timofte authored 5 days ago
102
        XCTAssertEqual(try countRows(in: "daily_type_aggregates WHERE observation_id = \(observationIDs[1]) AND visible_record_count = 1", at: url), 1)
Bogdan Timofte authored 2 weeks ago
103
        XCTAssertFalse(try tableExists("archive_samples", at: url))
104
    }
105

            
Bogdan Timofte authored 4 days ago
106
    func testUnchangedVerificationCopiesPreviousSummaryWithoutNewEvents() async throws {
107
        let url = databaseURL()
108
        let store = SQLiteHealthArchiveStore(databaseURL: url)
109
        let sample = makeStepCountSample()
110

            
111
        _ = try await store.upsertSamples([sample], observedAt: Date(timeIntervalSince1970: 2_000))
112
        try await store.markVerification(sampleType: sample.sampleType, verifiedAt: Date(timeIntervalSince1970: 2_060))
113
        let unchangedObservationID = try await store.beginObservation(
114
            observedAt: Date(timeIntervalSince1970: 2_120),
115
            triggerReason: "manual",
116
            selectedTypeSetHash: "selected-types"
117
        )
118
        try await store.markUnchangedVerification(
119
            sampleType: sample.sampleType,
120
            verifiedAt: Date(timeIntervalSince1970: 2_120),
121
            observationID: unchangedObservationID
122
        )
123
        try await store.finishObservation(
124
            observationID: unchangedObservationID,
125
            status: "completed",
126
            endedAt: Date(timeIntervalSince1970: 2_121)
127
        )
128

            
129
        XCTAssertEqual(try countRows(in: "sample_observation_events", at: url), 1)
130
        XCTAssertEqual(try countRows(in: "observation_type_runs WHERE observation_id = \(unchangedObservationID) AND inserted_event_count = 0 AND deleted_event_count = 0 AND verified_visible_count = 1", at: url), 1)
131
        XCTAssertEqual(try countRows(in: "observation_type_summaries WHERE observation_id = \(unchangedObservationID) AND visible_record_count = 1 AND appeared_count = 0 AND disappeared_count = 0 AND representation_changed_count = 0", at: url), 1)
132
        XCTAssertEqual(try countRows(in: "daily_type_aggregates WHERE observation_id = \(unchangedObservationID) AND visible_record_count = 1", at: url), 1)
133
        XCTAssertFalse(try tableExists("archive_samples", at: url))
134
    }
135

            
Bogdan Timofte authored 2 weeks ago
136
    func testDiffSummaryAndRecordsBetweenObservationsUseSQLVisibility() async throws {
137
        let url = databaseURL()
138
        let store = SQLiteHealthArchiveStore(databaseURL: url)
139
        let firstSample = makeStepCountSample(value: 42, start: 1_000)
140
        let secondSample = makeStepCountSample(value: 7, start: 2_000)
141
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
142

            
143
        _ = try await store.upsertSamples([firstSample], observedAt: Date(timeIntervalSince1970: 3_000))
144
        _ = try await store.upsertSamples([firstSample, secondSample], observedAt: Date(timeIntervalSince1970: 3_060))
145
        try await store.recordDisappearance(
146
            sampleUUIDHash: HashService.sampleUUIDHash(firstSample.uuid.uuidString),
147
            sampleTypeIdentifier: typeIdentifier,
148
            observedMissingAt: Date(timeIntervalSince1970: 3_120)
149
        )
150
        let observationIDs = try observationIDs(at: url)
151
        XCTAssertEqual(observationIDs.count, 3)
152

            
153
        let appearedSummary = try await store.diffSummary(HealthArchiveDiffRequest(
154
            fromObservationID: observationIDs[0],
155
            toObservationID: observationIDs[1],
156
            sampleTypeIdentifier: typeIdentifier
157
        ))
158
        let appearedRecords = try await store.diffRecords(HealthArchiveDiffRecordRequest(
159
            fromObservationID: observationIDs[0],
160
            toObservationID: observationIDs[1],
161
            sampleTypeIdentifier: typeIdentifier,
162
            kind: .appeared,
163
            limit: 10
164
        ))
165
        let disappearedSummary = try await store.diffSummary(HealthArchiveDiffRequest(
166
            fromObservationID: observationIDs[1],
167
            toObservationID: observationIDs[2],
168
            sampleTypeIdentifier: typeIdentifier
169
        ))
170
        let disappearedRecords = try await store.diffRecords(HealthArchiveDiffRecordRequest(
171
            fromObservationID: observationIDs[1],
172
            toObservationID: observationIDs[2],
173
            sampleTypeIdentifier: typeIdentifier,
174
            kind: .disappeared,
175
            limit: 10
176
        ))
177

            
178
        XCTAssertEqual(appearedSummary.appearedCount, 1)
179
        XCTAssertEqual(appearedSummary.disappearedCount, 0)
180
        XCTAssertEqual(appearedSummary.representationChangedCount, 0)
181
        XCTAssertEqual(appearedRecords.map(\.displayValue), ["7.0 count"])
182
        XCTAssertEqual(disappearedSummary.appearedCount, 0)
183
        XCTAssertEqual(disappearedSummary.disappearedCount, 1)
184
        XCTAssertEqual(disappearedSummary.representationChangedCount, 0)
185
        XCTAssertEqual(disappearedRecords.map(\.displayValue), ["42.0 count"])
186
        XCTAssertNotNil(disappearedRecords.first?.disappearedAt)
Bogdan Timofte authored a week ago
187

            
188
        let exportRequest = HealthArchiveReportRequest(
189
            reportID: UUID(),
190
            title: "Disappeared Step Count Export",
191
            typeIdentifierFilter: typeIdentifier,
192
            diffFromObservationID: observationIDs[1],
193
            diffToObservationID: observationIDs[2],
194
            diffKind: .disappeared
195
        )
196
        let exportPreview = try await store.exportPreview(exportRequest)
197
        let exportURL = try await store.exportReport(exportRequest)
198

            
199
        XCTAssertEqual(exportPreview.estimatedRecordCount, 1)
200
        XCTAssertTrue(FileManager.default.fileExists(atPath: exportURL.path))
201
        XCTAssertEqual(try countRows(in: "export_manifests WHERE record_count = 1 AND from_observation_id = \(observationIDs[1]) AND to_observation_id = \(observationIDs[2])", at: url), 1)
202
    }
203

            
204
    func testExportPreviewAndReportUseSQLiteManifest() async throws {
205
        let url = databaseURL()
206
        let store = SQLiteHealthArchiveStore(databaseURL: url)
207
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
208
        let samples = [
209
            makeStepCountSample(value: 42, start: 1_000),
210
            makeStepCountSample(value: 7, start: 2_000)
211
        ]
212

            
213
        _ = try await store.upsertSamples(samples, observedAt: Date(timeIntervalSince1970: 3_000))
214

            
215
        let request = HealthArchiveReportRequest(
216
            reportID: UUID(),
217
            title: "Step Count Export",
218
            typeIdentifierFilter: typeIdentifier
219
        )
220
        let preview = try await store.exportPreview(request)
221
        let exportURL = try await store.exportReport(request)
222

            
223
        XCTAssertEqual(preview.estimatedRecordCount, 2)
224
        XCTAssertTrue(FileManager.default.fileExists(atPath: exportURL.path))
225
        XCTAssertEqual(try countRows(in: "export_manifests", at: url), 1)
226
        XCTAssertEqual(try countRows(in: "export_manifests WHERE record_count = 2", at: url), 1)
227

            
228
        let cache = try CoreDataArchiveCacheStore(inMemory: true)
229
        let rebuild = try cache.rebuild(fromArchiveAt: url)
230
        XCTAssertEqual(rebuild.exportManifestRows, 1)
Bogdan Timofte authored 2 weeks ago
231
    }
232

            
Bogdan Timofte authored 2 weeks ago
233
    func testGroupedObservationKeepsPageWritesDeletesAndVerificationTogether() async throws {
234
        let url = databaseURL()
235
        let store = SQLiteHealthArchiveStore(databaseURL: url)
236
        let firstSample = makeStepCountSample(value: 42, start: 1_000)
237
        let secondSample = makeStepCountSample(value: 7, start: 2_000)
238
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
239

            
240
        _ = try await store.upsertSamples([firstSample], observedAt: Date(timeIntervalSince1970: 3_000))
241
        let observationID = try await store.beginObservation(
242
            observedAt: Date(timeIntervalSince1970: 3_060),
243
            triggerReason: "manual",
244
            selectedTypeSetHash: "selected-types"
245
        )
246
        _ = try await store.upsertSamples(
247
            [secondSample],
248
            observedAt: Date(timeIntervalSince1970: 3_060),
249
            observationID: observationID
250
        )
251
        try await store.recordDisappearance(
252
            sampleUUIDHash: HashService.sampleUUIDHash(firstSample.uuid.uuidString),
253
            sampleTypeIdentifier: typeIdentifier,
254
            observedMissingAt: Date(timeIntervalSince1970: 3_060),
255
            observationID: observationID
256
        )
Bogdan Timofte authored a week ago
257
        XCTAssertEqual(
258
            try countRows(
259
                in: "observation_type_summaries WHERE observation_id = \(observationID)",
260
                at: url
261
            ),
262
            0
263
        )
264
        XCTAssertEqual(
265
            try countRows(
266
                in: "daily_type_aggregates WHERE observation_id = \(observationID)",
267
                at: url
268
            ),
269
            0
270
        )
Bogdan Timofte authored 2 weeks ago
271
        try await store.markVerification(
272
            sampleType: secondSample.sampleType,
273
            verifiedAt: Date(timeIntervalSince1970: 3_060),
274
            observationID: observationID
275
        )
276
        try await store.finishObservation(
277
            observationID: observationID,
278
            status: "completed",
279
            endedAt: Date(timeIntervalSince1970: 3_070)
280
        )
281

            
282
        let observationIDs = try observationIDs(at: url)
283
        let summary = try await store.diffSummary(HealthArchiveDiffRequest(
284
            fromObservationID: observationIDs[0],
285
            toObservationID: observationID,
286
            sampleTypeIdentifier: typeIdentifier
287
        ))
288

            
289
        XCTAssertEqual(observationIDs.count, 2)
290
        XCTAssertEqual(summary.appearedCount, 1)
291
        XCTAssertEqual(summary.disappearedCount, 1)
292
        XCTAssertEqual(try countRows(in: "observations WHERE id = \(observationID) AND status = 'completed' AND selected_type_set_hash = 'selected-types'", at: url), 1)
293
        XCTAssertEqual(try countRows(in: "sample_observation_events WHERE observation_id = \(observationID)", at: url), 2)
294
        XCTAssertEqual(try countRows(in: "observation_type_runs WHERE observation_id = \(observationID) AND inserted_event_count = 1 AND deleted_event_count = 1 AND verified_visible_count = 1 AND status = 'completed'", at: url), 1)
Bogdan Timofte authored a week ago
295
        XCTAssertEqual(try countRows(in: "observation_type_summaries WHERE observation_id = \(observationID) AND visible_record_count = 1", at: url), 1)
296
        XCTAssertGreaterThan(try countRows(in: "daily_type_aggregates WHERE observation_id = \(observationID)", at: url), 0)
Bogdan Timofte authored 2 weeks ago
297
    }
298

            
Bogdan Timofte authored a week ago
299
    func testGroupedObservationCanBatchDeletedObjectsInSingleRun() async throws {
300
        let url = databaseURL()
301
        let store = SQLiteHealthArchiveStore(databaseURL: url)
302
        let samples = [
303
            makeStepCountSample(value: 42, start: 1_000),
304
            makeStepCountSample(value: 7, start: 2_000),
305
            makeStepCountSample(value: 9, start: 3_000)
306
        ]
307
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
308

            
309
        _ = try await store.upsertSamples(samples, observedAt: Date(timeIntervalSince1970: 4_000))
310
        let observationID = try await store.beginObservation(
311
            observedAt: Date(timeIntervalSince1970: 4_060),
312
            triggerReason: "manual",
313
            selectedTypeSetHash: "selected-types"
314
        )
315
        let deletedCount = try await store.recordDisappearances(
316
            sampleUUIDHashes: samples.map { HashService.sampleUUIDHash($0.uuid.uuidString) },
317
            sampleTypeIdentifier: typeIdentifier,
318
            observedMissingAt: Date(timeIntervalSince1970: 4_060),
319
            observationID: observationID
320
        )
321
        try await store.markVerification(
322
            sampleType: samples[0].sampleType,
323
            verifiedAt: Date(timeIntervalSince1970: 4_060),
324
            observationID: observationID
325
        )
326

            
327
        XCTAssertEqual(deletedCount, 3)
328
        XCTAssertEqual(try countRows(in: "sample_observation_events WHERE observation_id = \(observationID) AND event_kind = 'disappeared'", at: url), 3)
329
        XCTAssertEqual(try countRows(in: "observation_type_runs WHERE observation_id = \(observationID) AND deleted_event_count = 3 AND status = 'completed'", at: url), 1)
330
    }
331

            
Bogdan Timofte authored 2 weeks ago
332
    func testLargeSyntheticDiffReturnsCountsAndBoundedPages() async throws {
333
        let url = databaseURL()
334
        let store = SQLiteHealthArchiveStore(databaseURL: url)
335
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
336
        let initialCount = 1_200
337
        let appearedCount = 180
338
        let disappearedCount = 160
339
        let pageSize = 25
340
        let initialSamples = makeStepCountSamples(count: initialCount, startIndex: 0)
341
        let appearedSamples = makeStepCountSamples(count: appearedCount, startIndex: initialCount)
342

            
343
        _ = try await store.upsertSamples(initialSamples, observedAt: Date(timeIntervalSince1970: 1_000_000))
344
        _ = try await store.upsertSamples(initialSamples + appearedSamples, observedAt: Date(timeIntervalSince1970: 1_000_060))
345
        for (offset, sample) in initialSamples.prefix(disappearedCount).enumerated() {
346
            try await store.recordDisappearance(
347
                sampleUUIDHash: HashService.sampleUUIDHash(sample.uuid.uuidString),
348
                sampleTypeIdentifier: typeIdentifier,
349
                observedMissingAt: Date(timeIntervalSince1970: 1_000_120 + Double(offset))
350
            )
351
        }
352

            
353
        let observationIDs = try observationIDs(at: url)
354
        let firstObservationID = try XCTUnwrap(observationIDs.first)
355
        let lastObservationID = try XCTUnwrap(observationIDs.last)
356
        let queryStartedAt = Date()
357
        let summary = try await store.diffSummary(HealthArchiveDiffRequest(
358
            fromObservationID: firstObservationID,
359
            toObservationID: lastObservationID,
360
            sampleTypeIdentifier: typeIdentifier
361
        ))
362
        let firstPage = try await store.diffRecords(HealthArchiveDiffRecordRequest(
363
            fromObservationID: firstObservationID,
364
            toObservationID: lastObservationID,
365
            sampleTypeIdentifier: typeIdentifier,
366
            kind: .appeared,
367
            limit: pageSize
368
        ))
369
        let firstPageLastRecord = try XCTUnwrap(firstPage.last)
370
        let secondPage = try await store.diffRecords(HealthArchiveDiffRecordRequest(
371
            fromObservationID: firstObservationID,
372
            toObservationID: lastObservationID,
373
            sampleTypeIdentifier: typeIdentifier,
374
            kind: .appeared,
375
            afterCursor: RecordCursor(
376
                startDate: firstPageLastRecord.startDate,
377
                strictFingerprint: firstPageLastRecord.strictFingerprint
378
            ),
379
            limit: pageSize
380
        ))
381
        let queryElapsedSeconds = Date().timeIntervalSince(queryStartedAt)
382

            
383
        XCTAssertEqual(summary.appearedCount, appearedCount)
384
        XCTAssertEqual(summary.disappearedCount, disappearedCount)
385
        XCTAssertEqual(summary.representationChangedCount, 0)
386
        XCTAssertEqual(firstPage.count, pageSize)
387
        XCTAssertEqual(secondPage.count, pageSize)
388
        XCTAssertTrue(Set(firstPage.map(\.id)).isDisjoint(with: Set(secondPage.map(\.id))))
389
        XCTAssertLessThan(queryElapsedSeconds, 10)
390
    }
391

            
Bogdan Timofte authored 2 weeks ago
392
    func testLargeSyntheticDiffQueryMetrics() throws {
393
        let url = databaseURL()
394
        let store = SQLiteHealthArchiveStore(databaseURL: url)
395
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
396
        let initialCount = 900
397
        let appearedCount = 120
398
        let disappearedCount = 100
399
        let pageSize = 25
400
        let initialSamples = makeStepCountSamples(count: initialCount, startIndex: 0)
401
        let appearedSamples = makeStepCountSamples(count: appearedCount, startIndex: initialCount)
402

            
403
        try waitForArchiveOperation {
404
            _ = try await store.upsertSamples(initialSamples, observedAt: Date(timeIntervalSince1970: 1_200_000))
405
            _ = try await store.upsertSamples(initialSamples + appearedSamples, observedAt: Date(timeIntervalSince1970: 1_200_060))
406
            for (offset, sample) in initialSamples.prefix(disappearedCount).enumerated() {
407
                try await store.recordDisappearance(
408
                    sampleUUIDHash: HashService.sampleUUIDHash(sample.uuid.uuidString),
409
                    sampleTypeIdentifier: typeIdentifier,
410
                    observedMissingAt: Date(timeIntervalSince1970: 1_200_120 + Double(offset))
411
                )
412
            }
413
        }
414

            
415
        let observationIDs = try observationIDs(at: url)
416
        let firstObservationID = try XCTUnwrap(observationIDs.first)
417
        let lastObservationID = try XCTUnwrap(observationIDs.last)
418
        let options = XCTMeasureOptions()
419
        options.iterationCount = 3
420

            
421
        measure(metrics: [XCTClockMetric(), XCTMemoryMetric()], options: options) {
422
            do {
423
                let result = try waitForArchiveOperation {
424
                    let summary = try await store.diffSummary(HealthArchiveDiffRequest(
425
                        fromObservationID: firstObservationID,
426
                        toObservationID: lastObservationID,
427
                        sampleTypeIdentifier: typeIdentifier
428
                    ))
429
                    let records = try await store.diffRecords(HealthArchiveDiffRecordRequest(
430
                        fromObservationID: firstObservationID,
431
                        toObservationID: lastObservationID,
432
                        sampleTypeIdentifier: typeIdentifier,
433
                        kind: .appeared,
434
                        limit: pageSize
435
                    ))
436
                    return (summary, records)
437
                }
438
                XCTAssertEqual(result.0.appearedCount, appearedCount)
439
                XCTAssertEqual(result.0.disappearedCount, disappearedCount)
440
                XCTAssertEqual(result.1.count, pageSize)
441
            } catch {
442
                XCTFail("Measured archive query failed: \(error)")
443
            }
444
        }
445
    }
446

            
Bogdan Timofte authored 2 weeks ago
447
    func testAggregateComparisonUsesMaterializedDailyAggregates() async throws {
448
        let url = databaseURL()
449
        let store = SQLiteHealthArchiveStore(databaseURL: url)
450
        let firstSample = makeStepCountSample(value: 42, start: 1_000)
451
        let secondSample = makeStepCountSample(value: 7, start: 2_000)
452
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
453

            
454
        _ = try await store.upsertSamples([firstSample], observedAt: Date(timeIntervalSince1970: 3_000))
455
        _ = try await store.upsertSamples([firstSample, secondSample], observedAt: Date(timeIntervalSince1970: 3_060))
456
        let observationIDs = try observationIDs(at: url)
457
        XCTAssertEqual(observationIDs.count, 2)
458

            
459
        let rows = try await store.aggregateComparison(HealthArchiveAggregateComparisonRequest(
460
            fromObservationID: observationIDs[0],
461
            toObservationID: observationIDs[1],
462
            sampleTypeIdentifier: typeIdentifier,
463
            limit: 10
464
        ))
465

            
466
        XCTAssertEqual(rows.count, 1)
467
        XCTAssertEqual(rows.first?.sampleTypeIdentifier, typeIdentifier)
468
        XCTAssertEqual(rows.first?.fromVisibleRecordCount, 1)
469
        XCTAssertEqual(rows.first?.toVisibleRecordCount, 2)
470
        XCTAssertEqual(rows.first?.visibleRecordDelta, 1)
471
        XCTAssertEqual(rows.first?.fromValueSum, 42)
472
        XCTAssertEqual(rows.first?.toValueSum, 49)
473
        XCTAssertEqual(rows.first?.valueSumDelta, 7)
474
    }
475

            
Bogdan Timofte authored 2 weeks ago
476
    func testSourceProvenanceBreakdownReturnsVisibleSourceRows() async throws {
477
        let url = databaseURL()
478
        let store = SQLiteHealthArchiveStore(databaseURL: url)
479
        let sample = makeStepCountSample(value: 42, start: 1_000)
480
        let typeIdentifier = sample.sampleType.identifier
481

            
482
        _ = try await store.upsertSamples([sample], observedAt: Date(timeIntervalSince1970: 3_000))
483

            
484
        let rows = try await store.sourceProvenanceBreakdown(HealthArchiveSourceProvenanceRequest(
485
            visibleAtObservationID: nil,
486
            sampleTypeIdentifier: typeIdentifier,
487
            limit: 10
488
        ))
489

            
490
        XCTAssertEqual(rows.count, 1)
491
        XCTAssertEqual(rows.first?.sampleTypeIdentifier, typeIdentifier)
492
        XCTAssertEqual(rows.first?.visibleRecordCount, 1)
493
        XCTAssertEqual(rows.first?.valueSum, 42)
494
        XCTAssertEqual(rows.first?.sourceBundleIdentifier, sample.sourceRevision.source.bundleIdentifier)
495
    }
496

            
497
    func testConsolidationEvidenceClassifiesStableCountDropAsConsolidationLikely() async throws {
498
        let url = databaseURL()
499
        let store = SQLiteHealthArchiveStore(databaseURL: url)
500
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
501

            
502
        let firstOldSample = makeStepCountSample(value: 10, start: 1_000, end: 1_500)
503
        let secondOldSample = makeStepCountSample(value: 20, start: 1_600, end: 2_000)
504
        let consolidatedSample = makeStepCountSample(value: 30, start: 1_000, end: 2_500)
505

            
506
        _ = try await store.upsertSamples([firstOldSample, secondOldSample], observedAt: Date(timeIntervalSince1970: 3_000))
507
        _ = try await store.upsertSamples([consolidatedSample], observedAt: Date(timeIntervalSince1970: 3_060))
508
        try await store.recordDisappearance(sampleUUIDHash: HashService.sampleUUIDHash(firstOldSample.uuid.uuidString), sampleTypeIdentifier: typeIdentifier, observedMissingAt: Date(timeIntervalSince1970: 3_120))
509
        try await store.recordDisappearance(sampleUUIDHash: HashService.sampleUUIDHash(secondOldSample.uuid.uuidString), sampleTypeIdentifier: typeIdentifier, observedMissingAt: Date(timeIntervalSince1970: 3_180))
510

            
511
        let observationIDs = try observationIDs(at: url)
512
        XCTAssertEqual(observationIDs.count, 4)
513

            
514
        let rows = try await store.consolidationEvidence(HealthArchiveConsolidationEvidenceRequest(
515
            fromObservationID: observationIDs[0],
516
            toObservationID: observationIDs[observationIDs.count - 1],
517
            sampleTypeIdentifier: typeIdentifier
518
        ))
519

            
520
        XCTAssertEqual(rows.count, 1)
521
        XCTAssertEqual(rows.first?.sampleTypeIdentifier, typeIdentifier)
522
        XCTAssertEqual(rows.first?.disappearedCount, 2)
523
        XCTAssertEqual(rows.first?.appearedCount, 1)
524
        XCTAssertEqual(rows.first?.fromVisibleRecordCount, 2)
525
        XCTAssertEqual(rows.first?.toVisibleRecordCount, 1)
526
        XCTAssertEqual(rows.first?.fromValueSum, 30)
527
        XCTAssertEqual(rows.first?.toValueSum, 30)
528
        XCTAssertEqual(rows.first?.label, "consolidation_likely")
529
        XCTAssertTrue(rows.first?.sourceCompatible == true)
530
    }
531

            
532
    func testConsolidationEvidenceClassifiesDenseStableShiftAsAggregateChanged() async throws {
533
        let url = databaseURL()
534
        let store = SQLiteHealthArchiveStore(databaseURL: url)
535
        let typeIdentifier = HKQuantityTypeIdentifier.stepCount.rawValue
536

            
537
        let firstOldSample = makeStepCountSample(value: 10, start: 1_000, end: 1_500)
538
        let secondOldSample = makeStepCountSample(value: 20, start: 5_000, end: 6_000)
539
        let denseSample = makeStepCountSample(value: 30, start: 1_000, end: 1_200)
540

            
541
        _ = try await store.upsertSamples([firstOldSample, secondOldSample], observedAt: Date(timeIntervalSince1970: 3_000))
542
        _ = try await store.upsertSamples([denseSample], observedAt: Date(timeIntervalSince1970: 3_060))
543
        try await store.recordDisappearance(sampleUUIDHash: HashService.sampleUUIDHash(firstOldSample.uuid.uuidString), sampleTypeIdentifier: typeIdentifier, observedMissingAt: Date(timeIntervalSince1970: 3_120))
544
        try await store.recordDisappearance(sampleUUIDHash: HashService.sampleUUIDHash(secondOldSample.uuid.uuidString), sampleTypeIdentifier: typeIdentifier, observedMissingAt: Date(timeIntervalSince1970: 3_180))
545

            
546
        let observationIDs = try observationIDs(at: url)
547
        XCTAssertEqual(observationIDs.count, 4)
548

            
549
        let rows = try await store.consolidationEvidence(HealthArchiveConsolidationEvidenceRequest(
550
            fromObservationID: observationIDs[0],
551
            toObservationID: observationIDs[observationIDs.count - 1],
552
            sampleTypeIdentifier: typeIdentifier
553
        ))
554

            
555
        XCTAssertEqual(rows.count, 1)
556
        XCTAssertEqual(rows.first?.label, "aggregate_changed")
557
        XCTAssertEqual(rows.first?.fromVisibleRecordCount, 2)
558
        XCTAssertEqual(rows.first?.toVisibleRecordCount, 1)
559
        XCTAssertEqual(rows.first?.fromValueSum, 30)
560
        XCTAssertEqual(rows.first?.toValueSum, 30)
561
    }
562

            
Bogdan Timofte authored 2 weeks ago
563
    private func databaseURL() -> URL {
564
        temporaryDirectory.appending(path: "Archive.sqlite")
565
    }
566

            
567
    private func createPrototypeDatabase(at url: URL) throws {
568
        var db: OpaquePointer?
569
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
570
            sqlite3_close(db)
571
            XCTFail("Could not create prototype database")
572
            return
573
        }
574
        defer { sqlite3_close(db) }
575
        let status = sqlite3_exec(db, "CREATE TABLE archive_samples (id TEXT PRIMARY KEY)", nil, nil, nil)
576
        XCTAssertEqual(status, SQLITE_OK)
577
    }
578

            
579
    private func makeStepCountSample() -> HKQuantitySample {
Bogdan Timofte authored 2 weeks ago
580
        makeStepCountSample(value: 42, start: 1_000)
581
    }
582

            
Bogdan Timofte authored 2 weeks ago
583
    private func makeStepCountSample(value: Double, start: TimeInterval, end: TimeInterval? = nil) -> HKQuantitySample {
Bogdan Timofte authored 2 weeks ago
584
        let quantityType = HKQuantityType.quantityType(forIdentifier: .stepCount)!
Bogdan Timofte authored 2 weeks ago
585
        let quantity = HKQuantity(unit: .count(), doubleValue: value)
586
        let startDate = Date(timeIntervalSince1970: start)
Bogdan Timofte authored 2 weeks ago
587
        let endDate = Date(timeIntervalSince1970: end ?? (start + 300))
Bogdan Timofte authored 2 weeks ago
588
        return HKQuantitySample(type: quantityType, quantity: quantity, start: startDate, end: endDate)
Bogdan Timofte authored 2 weeks ago
589
    }
590

            
Bogdan Timofte authored 2 weeks ago
591
    private func makeStepCountSamples(count: Int, startIndex: Int) -> [HKQuantitySample] {
592
        (0..<count).map { offset in
593
            let index = startIndex + offset
594
            return makeStepCountSample(
595
                value: Double((index % 97) + 1),
596
                start: 10_000 + Double(index * 600)
597
            )
598
        }
599
    }
600

            
Bogdan Timofte authored 2 weeks ago
601
    private func waitForArchiveOperation<T>(_ operation: @escaping () async throws -> T) throws -> T {
602
        let expectation = expectation(description: "archive operation")
603
        let box = AsyncResultBox<T>()
604

            
605
        Task {
606
            do {
607
                box.result = .success(try await operation())
608
            } catch {
609
                box.result = .failure(error)
610
            }
611
            expectation.fulfill()
612
        }
613

            
614
        wait(for: [expectation], timeout: 20)
615
        return try XCTUnwrap(box.result).get()
616
    }
617

            
Bogdan Timofte authored 2 weeks ago
618
    private func countRows(in tableName: String, at url: URL) throws -> Int {
619
        var db: OpaquePointer?
620
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
621
            sqlite3_close(db)
622
            XCTFail("Could not open test database")
623
            return 0
624
        }
625
        defer { sqlite3_close(db) }
626

            
627
        var statement: OpaquePointer?
628
        guard sqlite3_prepare_v2(db, "SELECT COUNT(*) FROM \(tableName)", -1, &statement, nil) == SQLITE_OK else {
629
            sqlite3_finalize(statement)
630
            XCTFail("Could not prepare count query")
631
            return 0
632
        }
633
        defer { sqlite3_finalize(statement) }
634

            
635
        guard sqlite3_step(statement) == SQLITE_ROW else {
636
            return 0
637
        }
638
        return Int(sqlite3_column_int(statement, 0))
639
    }
640

            
Bogdan Timofte authored 2 weeks ago
641
    private func tableExists(_ tableName: String, at url: URL) throws -> Bool {
642
        var db: OpaquePointer?
643
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
644
            sqlite3_close(db)
645
            XCTFail("Could not open test database")
646
            return false
647
        }
648
        defer { sqlite3_close(db) }
649

            
650
        let sql = """
651
        SELECT 1
652
        FROM sqlite_master
653
        WHERE type = 'table' AND name = ?
654
        LIMIT 1
655
        """
656
        var statement: OpaquePointer?
657
        guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else {
658
            sqlite3_finalize(statement)
659
            XCTFail("Could not prepare table existence query")
660
            return false
661
        }
662
        defer { sqlite3_finalize(statement) }
663

            
664
        sqlite3_bind_text(statement, 1, tableName, -1, unsafeBitCast(-1, to: sqlite3_destructor_type.self))
665
        return sqlite3_step(statement) == SQLITE_ROW
666
    }
667

            
Bogdan Timofte authored 2 weeks ago
668
    private func observationIDs(at url: URL) throws -> [Int64] {
669
        var db: OpaquePointer?
670
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
671
            sqlite3_close(db)
672
            XCTFail("Could not open test database")
673
            return []
674
        }
675
        defer { sqlite3_close(db) }
676

            
677
        var statement: OpaquePointer?
678
        guard sqlite3_prepare_v2(db, "SELECT id FROM observations ORDER BY id", -1, &statement, nil) == SQLITE_OK else {
679
            sqlite3_finalize(statement)
680
            XCTFail("Could not prepare observation query")
681
            return []
682
        }
683
        defer { sqlite3_finalize(statement) }
684

            
685
        var ids: [Int64] = []
686
        while sqlite3_step(statement) == SQLITE_ROW {
687
            ids.append(sqlite3_column_int64(statement, 0))
688
        }
689
        return ids
690
    }
691

            
Bogdan Timofte authored 2 weeks ago
692
    private func sampleVersionDebugRows(at url: URL) throws -> String {
693
        var db: OpaquePointer?
694
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
695
            sqlite3_close(db)
696
            return "could not open database"
697
        }
698
        defer { sqlite3_close(db) }
699

            
700
        let sql = """
701
        SELECT v.payload_hash, v.start_date, v.end_date, v.value_kind, v.numeric_value, v.unit,
702
               v.source_revision_id, src.bundle_identifier, sr.product_type, sr.version,
703
               sr.operating_system_version, v.hk_device_id, v.metadata_id
704
        FROM sample_versions v
705
        LEFT JOIN source_revisions sr ON sr.id = v.source_revision_id
706
        LEFT JOIN sources src ON src.id = sr.source_id
707
        ORDER BY v.id
708
        """
709
        var statement: OpaquePointer?
710
        guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else {
711
            sqlite3_finalize(statement)
712
            return "could not prepare version debug query"
713
        }
714
        defer { sqlite3_finalize(statement) }
715

            
716
        var rows: [String] = []
717
        while sqlite3_step(statement) == SQLITE_ROW {
718
            rows.append((0..<13).map { index in
719
                if sqlite3_column_type(statement, Int32(index)) == SQLITE_NULL {
720
                    return "null"
721
                }
722
                if let text = sqlite3_column_text(statement, Int32(index)) {
723
                    return String(cString: text)
724
                }
725
                return "\(sqlite3_column_double(statement, Int32(index)))"
726
            }.joined(separator: "|"))
727
        }
728
        return rows.joined(separator: "\n")
729
    }
Bogdan Timofte authored 6 days ago
730

            
731
    private func visibilityRangeDebugRows(at url: URL) throws -> String {
732
        var db: OpaquePointer?
733
        guard sqlite3_open_v2(url.path, &db, SQLITE_OPEN_READONLY | SQLITE_OPEN_FULLMUTEX, nil) == SQLITE_OK else {
734
            sqlite3_close(db)
735
            return "could not open database"
736
        }
737
        defer { sqlite3_close(db) }
738

            
739
        let sql = """
740
        SELECT sample_id, version_id, first_observation_id, last_observation_id, first_seen_at, last_seen_at
741
        FROM sample_visibility_ranges
742
        ORDER BY sample_id, version_id, first_observation_id
743
        """
744
        var statement: OpaquePointer?
745
        guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else {
746
            sqlite3_finalize(statement)
747
            return "could not prepare visibility debug query"
748
        }
749
        defer { sqlite3_finalize(statement) }
750

            
751
        var rows: [String] = []
752
        while sqlite3_step(statement) == SQLITE_ROW {
753
            rows.append((0..<6).map { index in
754
                if sqlite3_column_type(statement, Int32(index)) == SQLITE_NULL {
755
                    return "null"
756
                }
757
                if let text = sqlite3_column_text(statement, Int32(index)) {
758
                    return String(cString: text)
759
                }
760
                return "\(sqlite3_column_double(statement, Int32(index)))"
761
            }.joined(separator: "|"))
762
        }
763
        return rows.joined(separator: "\n")
764
    }
Bogdan Timofte authored 2 weeks ago
765
}