LOOP-1201 Export of Critical Event Logs

LoopKit/CarbKit/CachedCarbObject+CoreDataProperties.swift

@@ -34,3 +34,44 @@ extension CachedCarbObject {
     @NSManaged public var anchorKey: Int64
 
 }
+
+extension CachedCarbObject: Encodable {
+    public func encode(to encoder: Encoder) throws {
+        var container = encoder.container(keyedBy: CodingKeys.self)
+        try container.encodeIfPresent(absorptionTime, forKey: .absorptionTime)
+        try container.encode(createdByCurrentApp, forKey: .createdByCurrentApp)
+        try container.encodeIfPresent(foodType, forKey: .foodType)
+        try container.encode(grams, forKey: .grams)
+        try container.encode(startDate, forKey: .startDate)
+        try container.encodeIfPresent(uuid, forKey: .uuid)
+        try container.encodeIfPresent(provenanceIdentifier, forKey: .provenanceIdentifier)
+        try container.encodeIfPresent(syncIdentifier, forKey: .syncIdentifier)
+        try container.encodeIfPresent(syncVersion, forKey: .syncVersion)
+        try container.encodeIfPresent(userCreatedDate, forKey: .userCreatedDate)
+        try container.encodeIfPresent(userUpdatedDate, forKey: .userUpdatedDate)
+        try container.encodeIfPresent(userDeletedDate, forKey: .userDeletedDate)
+        try container.encodeIfPresent(operation, forKey: .operation)
+        try container.encodeIfPresent(addedDate, forKey: .addedDate)
+        try container.encodeIfPresent(supercededDate, forKey: .supercededDate)
+        try container.encode(anchorKey, forKey: .anchorKey)
+    }
+
+    private enum CodingKeys: String, CodingKey {
+        case absorptionTime
+        case createdByCurrentApp
+        case foodType
+        case grams
+        case startDate
+        case uuid
+        case provenanceIdentifier
+        case syncIdentifier
+        case syncVersion
+        case userCreatedDate
+        case userUpdatedDate
+        case userDeletedDate
+        case operation
+        case addedDate
+        case supercededDate
+        case anchorKey
+    }
+}

LoopKit/CarbKit/CarbStore.swift

@@ -1294,6 +1294,86 @@ extension CarbStore {
     }
 }
 
+// MARK: - Critical Event Log Export
+
+extension CarbStore: CriticalEventLog {
+    private var exportProgressUnitCountPerObject: Int64 { 1 }
+    private var exportFetchLimit: Int { Int(criticalEventLogExportProgressUnitCountPerFetch / exportProgressUnitCountPerObject) }
+
+    public var exportName: String { "Carbs.json" }
+
+    public func exportProgressTotalUnitCount(startDate: Date, endDate: Date? = nil) -> Result<Int64, Error> {
+        var result: Result<Int64, Error>?
+
+        self.cacheStore.managedObjectContext.performAndWait {
+            do {
+                let request: NSFetchRequest<CachedCarbObject> = CachedCarbObject.fetchRequest()
+                request.predicate = self.exportDatePredicate(startDate: startDate, endDate: endDate)
+
+                let objectCount = try self.cacheStore.managedObjectContext.count(for: request)
+                result = .success(Int64(objectCount) * exportProgressUnitCountPerObject)
+            } catch let error {
+                result = .failure(error)
+            }
+        }
+
+        return result!
+    }
+
+    public func export(startDate: Date, endDate: Date, to stream: OutputStream, progress: Progress) -> Error? {
+        let encoder = JSONStreamEncoder(stream: stream)
+        var anchorKey: Int64 = 0
+        var fetching = true
+        var error: Error?
+
+        while fetching && error == nil {
+            self.cacheStore.managedObjectContext.performAndWait {
+                do {
+                    guard !progress.isCancelled else {
+                        throw CriticalEventLogError.cancelled
+                    }
+
+                    let request: NSFetchRequest<CachedCarbObject> = CachedCarbObject.fetchRequest()
+                    request.predicate = NSCompoundPredicate(andPredicateWithSubpredicates: [NSPredicate(format: "anchorKey > %d", anchorKey),
+                                                                                            self.exportDatePredicate(startDate: startDate, endDate: endDate)])
+                    request.sortDescriptors = [NSSortDescriptor(key: "anchorKey", ascending: true)]
+                    request.fetchLimit = self.exportFetchLimit
+
+                    let objects = try self.cacheStore.managedObjectContext.fetch(request)
+                    if objects.isEmpty {
+                        fetching = false
+                        return
+                    }
+
+                    try encoder.encode(objects)
+
+                    anchorKey = objects.last!.anchorKey
+
+                    progress.completedUnitCount += Int64(objects.count) * exportProgressUnitCountPerObject
+                } catch let fetchError {
+                    error = fetchError
+                }
+            }
+        }
+
+        if let closeError = encoder.close(), error == nil {
+            error = closeError
+        }
+
+        return error
+    }
+
+    private func exportDatePredicate(startDate: Date, endDate: Date? = nil) -> NSPredicate {
+        var addedDatePredicate = NSPredicate(format: "addedDate >= %@", startDate as NSDate)
+        var supercededDatePredicate = NSPredicate(format: "supercededDate >= %@", startDate as NSDate)
+        if let endDate = endDate {
+            addedDatePredicate = NSCompoundPredicate(andPredicateWithSubpredicates: [addedDatePredicate, NSPredicate(format: "addedDate < %@", endDate as NSDate)])
+            supercededDatePredicate = NSCompoundPredicate(andPredicateWithSubpredicates: [supercededDatePredicate, NSPredicate(format: "supercededDate < %@", endDate as NSDate)])
+        }
+        return NSCompoundPredicate(orPredicateWithSubpredicates: [addedDatePredicate, supercededDatePredicate])
+    }
+}
+
 // MARK: - Core Data (Bulk) - TEST ONLY
 
 extension CarbStore {
@@ -1306,16 +1386,14 @@ extension CarbStore {
         queue.async {
             var error: Error?
 
-            let date = Date()
-
             self.cacheStore.managedObjectContext.performAndWait {
                 do {
                     for entry in entries {
                         let syncIdentifier = try self.cacheStore.managedObjectContext.generateUniqueSyncIdentifier()
 
                         let object = CachedCarbObject(context: self.cacheStore.managedObjectContext)
                         object.create(from: entry,
-                                      on: date,
+                                      on: entry.date,
                                       provenanceIdentifier: self.provenanceIdentifier,
                                       syncIdentifier: syncIdentifier)
                     }

LoopKit/CriticalEventLog.swift

@@ -0,0 +1,41 @@
+//
+//  CriticalEventLog.swift
+//  LoopKit
+//
+//  Created by Darin Krauss on 7/15/20.
+//  Copyright © 2020 LoopKit Authors. All rights reserved.
+//
+
+import Foundation
+
+public protocol CriticalEventLog {
+
+    /// The name for the critical event log export.
+    var exportName: String { get }
+
+    /// Calculate the progress total unit count for the critical event log export for the specified date range.
+    ///
+    /// - Parameters:
+    ///   - startDate: The start date for the critical events to export.
+    ///   - endDate: The end date for the critical events to export. Optional. If not specified, default to now.
+    /// - Returns: An progress total unit count, or an error.
+    func exportProgressTotalUnitCount(startDate: Date, endDate: Date?) -> Result<Int64, Error>
+
+    /// Export the critical event log for the specified date range.
+    ///
+    /// - Parameters:
+    ///   - startDate: The start date for the critical events to export.
+    ///   - endDate: The end date for the critical events to export.
+    ///   - stream: The output stream to write the critical event log to. Typically writes JSON UTF-8 text.
+    ///   - progressor: The estimated duration progress to use to check if cancelled and report progress.
+    /// - Returns: Any error that occurs during the export, or nil if successful.
+    func export(startDate: Date, endDate: Date, to stream: OutputStream, progress: Progress) -> Error?
+}
+
+public enum CriticalEventLogError: Error {
+
+    /// The export was cancelled either by the user or the OS.
+    case cancelled
+}
+
+public let criticalEventLogExportProgressUnitCountPerFetch: Int64 = 250

LoopKit/DeviceManager/DeviceLog/DeviceLog.xcdatamodeld/DeviceCommsLog.xcdatamodel/contents

@@ -1,13 +1,17 @@
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<model type="com.apple.IDECoreDataModeler.DataModel" documentVersion="1.0" lastSavedToolsVersion="15400" systemVersion="19C57" minimumToolsVersion="Automatic" sourceLanguage="Swift" userDefinedModelVersionIdentifier="">
+<model type="com.apple.IDECoreDataModeler.DataModel" documentVersion="1.0" lastSavedToolsVersion="16119" systemVersion="19G2021" minimumToolsVersion="Automatic" sourceLanguage="Swift" userDefinedModelVersionIdentifier="">
     <entity name="Entry" representedClassName=".DeviceLogEntry" syncable="YES">
         <attribute name="deviceIdentifier" optional="YES" attributeType="String"/>
         <attribute name="managerIdentifier" attributeType="String"/>
         <attribute name="message" attributeType="String"/>
+        <attribute name="modificationCounter" attributeType="Integer 64" defaultValueString="0" usesScalarValueType="YES"/>
         <attribute name="timestamp" attributeType="Date" usesScalarValueType="NO"/>
         <attribute name="type" attributeType="String"/>
+        <fetchIndex name="byTimestampIndex">
+            <fetchIndexElement property="timestamp" type="Binary" order="ascending"/>
+        </fetchIndex>
     </entity>
     <elements>
-        <element name="Entry" positionX="-36" positionY="9" width="128" height="118"/>
+        <element name="Entry" positionX="-36" positionY="9" width="128" height="133"/>
     </elements>
 </model>

LoopKit/DeviceManager/DeviceLog/DeviceLogEntry+CoreDataClass.swift

@@ -28,4 +28,19 @@ class DeviceLogEntry: NSManagedObject {
         }
     }
 
+    var hasUpdatedModificationCounter: Bool { changedValues().keys.contains("modificationCounter") }
+
+    func updateModificationCounter() { setPrimitiveValue(managedObjectContext!.modificationCounter!, forKey: "modificationCounter") }
+
+    override func awakeFromInsert() {
+        super.awakeFromInsert()
+        updateModificationCounter()
+    }
+
+    override func willSave() {
+        if isUpdated && !hasUpdatedModificationCounter {
+            updateModificationCounter()
+        }
+        super.willSave()
+    }
 }

LoopKit/DeviceManager/DeviceLog/DeviceLogEntry+CoreDataProperties.swift

@@ -22,9 +22,31 @@ extension DeviceLogEntry {
     @NSManaged public var deviceIdentifier: String?
     @NSManaged public var message: String?
     @NSManaged public var timestamp: Date?
+    @NSManaged public var modificationCounter: Int64
 
 }
 
+extension DeviceLogEntry: Encodable {
+    public func encode(to encoder: Encoder) throws {
+        var container = encoder.container(keyedBy: CodingKeys.self)
+        try container.encodeIfPresent(type?.rawValue, forKey: .type)
+        try container.encodeIfPresent(managerIdentifier, forKey: .managerIdentifier)
+        try container.encodeIfPresent(deviceIdentifier, forKey: .deviceIdentifier)
+        try container.encodeIfPresent(message, forKey: .message)
+        try container.encodeIfPresent(timestamp, forKey: .timestamp)
+        try container.encode(modificationCounter, forKey: .modificationCounter)
+    }
+
+    private enum CodingKeys: String, CodingKey {
+        case type
+        case managerIdentifier
+        case deviceIdentifier
+        case message
+        case timestamp
+        case modificationCounter
+    }
+}
+
 extension DeviceLogEntry {
     func update(from entry: StoredDeviceLogEntry) {
         type = entry.type

LoopKit/DeviceManager/DeviceLog/PersistentDeviceLog.swift

@@ -117,6 +117,84 @@ public class PersistentDeviceLog {
     }
 }
 
+// MARK: - Critical Event Log Export
+
+extension PersistentDeviceLog: CriticalEventLog {
+    private var exportProgressUnitCountPerObject: Int64 { 1 }
+    private var exportFetchLimit: Int { Int(criticalEventLogExportProgressUnitCountPerFetch / exportProgressUnitCountPerObject) }
+
+    public var exportName: String { "DeviceLog.json" }
+
+    public func exportProgressTotalUnitCount(startDate: Date, endDate: Date? = nil) -> Result<Int64, Error> {
+        var result: Result<Int64, Error>?
+
+        self.managedObjectContext.performAndWait {
+            do {
+                let request: NSFetchRequest<DeviceLogEntry> = DeviceLogEntry.fetchRequest()
+                request.predicate = self.exportDatePredicate(startDate: startDate, endDate: endDate)
+
+                let objectCount = try self.managedObjectContext.count(for: request)
+                result = .success(Int64(objectCount) * exportProgressUnitCountPerObject)
+            } catch let error {
+                result = .failure(error)
+            }
+        }
+
+        return result!
+    }
+
+    public func export(startDate: Date, endDate: Date, to stream: OutputStream, progress: Progress) -> Error? {
+        let encoder = JSONStreamEncoder(stream: stream)
+        var modificationCounter: Int64 = 0
+        var fetching = true
+        var error: Error?
+
+        while fetching && error == nil {
+            self.managedObjectContext.performAndWait {
+                do {
+                    guard !progress.isCancelled else {
+                        throw CriticalEventLogError.cancelled
+                    }
+
+                    let request: NSFetchRequest<DeviceLogEntry> = DeviceLogEntry.fetchRequest()
+                    request.predicate = NSCompoundPredicate(andPredicateWithSubpredicates: [NSPredicate(format: "modificationCounter > %d", modificationCounter),
+                                                                                            self.exportDatePredicate(startDate: startDate, endDate: endDate)])
+                    request.sortDescriptors = [NSSortDescriptor(key: "modificationCounter", ascending: true)]
+                    request.fetchLimit = self.exportFetchLimit
+
+                    let objects = try self.managedObjectContext.fetch(request)
+                    if objects.isEmpty {
+                        fetching = false
+                        return
+                    }
+
+                    try encoder.encode(objects)
+
+                    modificationCounter = objects.last!.modificationCounter
+
+                    progress.completedUnitCount += Int64(objects.count) * exportProgressUnitCountPerObject
+                } catch let fetchError {
+                    error = fetchError
+                }
+            }
+        }
+
+        if let closeError = encoder.close(), error == nil {
+            error = closeError
+        }
+
+        return error
+    }
+
+    private func exportDatePredicate(startDate: Date, endDate: Date? = nil) -> NSPredicate {
+        var predicate = NSPredicate(format: "timestamp >= %@", startDate as NSDate)
+        if let endDate = endDate {
+            predicate = NSCompoundPredicate(andPredicateWithSubpredicates: [predicate, NSPredicate(format: "timestamp < %@", endDate as NSDate)])
+        }
+        return predicate
+    }
+}
+
 // MARK: - Core Data (Bulk) - TEST ONLY
 
 extension PersistentDeviceLog {

LoopKit/DeviceManager/PumpManagerStatus.swift

@@ -98,8 +98,8 @@ extension PumpManagerStatus: Codable {
         self.pumpBatteryChargeRemaining = try container.decodeIfPresent(Double.self, forKey: .pumpBatteryChargeRemaining)
         self.basalDeliveryState = try container.decode(BasalDeliveryState.self, forKey: .basalDeliveryState)
         self.bolusState = try container.decode(BolusState.self, forKey: .bolusState)
-        self.pumpStatusHighlight = try container.decode(PumpStatusHighlight.self, forKey: .pumpStatusHighlight)
-        self.pumpLifecycleProgress = try container.decode(PumpLifecycleProgress.self, forKey: .pumpLifecycleProgress)
+        self.pumpStatusHighlight = try container.decodeIfPresent(PumpStatusHighlight.self, forKey: .pumpStatusHighlight)
+        self.pumpLifecycleProgress = try container.decodeIfPresent(PumpLifecycleProgress.self, forKey: .pumpLifecycleProgress)
         self.deliveryIsUncertain = try container.decode(Bool.self, forKey: .deliveryIsUncertain)
     }
 
@@ -110,8 +110,8 @@ extension PumpManagerStatus: Codable {
         try container.encodeIfPresent(pumpBatteryChargeRemaining, forKey: .pumpBatteryChargeRemaining)
         try container.encode(basalDeliveryState, forKey: .basalDeliveryState)
         try container.encode(bolusState, forKey: .bolusState)
-        try container.encode(pumpStatusHighlight, forKey: .pumpStatusHighlight)
-        try container.encode(pumpLifecycleProgress, forKey: .pumpLifecycleProgress)
+        try container.encodeIfPresent(pumpStatusHighlight, forKey: .pumpStatusHighlight)
+        try container.encodeIfPresent(pumpLifecycleProgress, forKey: .pumpLifecycleProgress)
         try container.encode(deliveryIsUncertain, forKey: .deliveryIsUncertain)
     }
 

LoopKit/DosingDecisionObject+CoreDataClass.swift

@@ -8,17 +8,17 @@
 
 import CoreData
 
-class DosingDecisionObject: NSManagedObject {
+public class DosingDecisionObject: NSManagedObject {
     var hasUpdatedModificationCounter: Bool { changedValues().keys.contains("modificationCounter") }
 
     func updateModificationCounter() { setPrimitiveValue(managedObjectContext!.modificationCounter!, forKey: "modificationCounter") }
 
-    override func awakeFromInsert() {
+    public override func awakeFromInsert() {
         super.awakeFromInsert()
         updateModificationCounter()
     }
 
-    override func willSave() {
+    public override func willSave() {
         if isUpdated && !hasUpdatedModificationCounter {
             updateModificationCounter()
         }