How to properly store state information with SwiftUI & UIViewRepresentable? (MKMapView)

I have a design problem which was typically quite simple to remedy with UIKit but I'm having a hard time figuring out the best way to replicate that functionally with SwiftUI.

Essentially, I am tracking multiple users and displaying them on a map. Their locations are updated and when updateUIView gets called I just check to see if they have moved and move their associated annotation accordingly.

The problem comes in when I try to draw a polyline behind them to indicate they are on a trip. I want to store each of their current plots in a dictionary (activeTripPlots & activeTripOverlays) so that I may remove them (the paths) from the map easily when they need to be.

Each User has a currentTrip object that is working as expected, and returns nil if they are not on a trip. At that point i would obviously like to remove the path plotted behind them AND update the activeTripPlot dictionary when it is removed.

The setup below is not good, as I get a warning for modifying state in updateUIView. I just don't see exactly where I would put these variables? The Coordinator does not seem like the right place as it is information specific to what is actually rendered on the MapView

struct MapView: UIViewRepresentable {

    var mapView = MKMapView()

var contacts: [User]

var locatedContacts: [User] {
    return contacts.filter { $0.location != nil }
}

var annotations: [UserAnnotation] {
    return locatedContacts.map { UserAnnotation($0) }
}

// [ID : Coordinates for active trip of user currently plotted]
@State var activeTripPlots = [String : [CLLocationCoordinate2D]]()

// [ID : Polyline segments for active trip of user currently plotted]
@State var activeTripOverlays = [String : [MKPolyline]]()

...

func updateUIView(_ view: MKMapView, context: Context) {
    
    // Draw user map markers
    if view.annotations.count <= locatedContacts.count {
        print("New annotations update")
        view.removeAnnotations(view.annotations)
        view.addAnnotations(annotations)
    }
    
    // TODO: Instead of doing this here figure out how to tie User location to the annotation reactively
    for annotation in view.annotations {
        guard let userAnnotation = annotation as? UserAnnotation else { continue } // Will trip for self marker
        guard let contact = locatedContacts.first(where: { $0 == userAnnotation.user }) else { continue }
        let contactCoordinate = contact.location!.coreLocation.coordinate
        if userAnnotation.coordinate != contactCoordinate {
            moveAnnotation(userAnnotation, to: contactCoordinate, animated: true, view: view)
            print("Location updated for \(contact.firstName)")
        }
    }
}

func moveAnnotation(_ userAnnotation: UserAnnotation, to location: CLLocationCoordinate2D, animated: Bool, view: MKMapView) {
    UIView.animate(withDuration: animated ? 3 : 0, animations: {
        userAnnotation.coordinate = location
    })
    
    // Plot trip of contact we have received a location update from
    
    guard let contact = locatedContacts.first(where: { $0 == userAnnotation.user }) else { return }
    
    // If there is no current trip for the contact, remove overlays and stale data from past trip
    guard let contactUpdatedTrip = contact.currentTrip else {
        if let contactTripOverlays = activeTripOverlays[contact.Id] {
            view.removeOverlays(contactTripOverlays)
            activeTripOverlays[contact.Id] = [MKPolyline]()
        }
        activeTripPlots[contact.Id] = [CLLocationCoordinate2D]()
        return
    }

    // There exists a current trip for the updated contact, so we will plot it

    let contactUpdatedTripCoords = contactUpdatedTrip.map { $0.coreLocation.coordinate }
    
    // The currently plotted coordinates / overlays on our map view
    let contactPlot = activeTripPlots[contact.Id] ?? [CLLocationCoordinate2D]()
    let contactTripOverlays = activeTripOverlays[contact.Id] ?? [MKPolyline]()
    
    // If the trip currently plotted for a user is missing newer updates held in the contact, plot them
    // TOOD: Should use a trip identifier in the future to prevent different trips from being compared and plotted by this as one
    if contactPlot.count < contactUpdatedTripCoords.count {
        let newSegmentRange = (contactPlot.count == 0 ? 0 : contactPlot.count - 1, contactUpdatedTripCoords.count - 1)
        var newSegment = Array(contactUpdatedTripCoords[newSegmentRange.0 ... newSegmentRange.1])
        let polyline = MKPolyline(coordinates: &newSegment, count: newSegment.count)
        view.addOverlay(polyline) // TODO: do this in the same animation as the annotation movement IFF segment length == 2
        activeTripPlots[contact.Id] = contactUpdatedTripCoords // Update local plot store so we know if there are new changes in future updates
        activeTripOverlays[contact.Id] = contactTripOverlays + [polyline]
        return
    }
    
    // Currently plotted trip has either ended or is stale data
    if contactPlot.count > contactUpdatedTripCoords.count {
        view.removeOverlays(contactTripOverlays)
        activeTripOverlays[contact.Id] = [MKPolyline]()
        activeTripPlots[contact.Id] = [CLLocationCoordinate2D]()
    }
}