Skip to content

fix(tracing): Resync informer cache in root owner retries #14690

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: sfleen/root-owner-3
Choose a base branch
from
Open

fix(tracing): Resync informer cache in root owner retries #14690

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 12 additions & 21 deletions controller/k8s/metadata_api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,15 +279,7 @@ func (api *MetadataAPI) GetOwnerKindAndName(ctx context.Context, pod *corev1.Pod
// Currently, this is limited to Jobs (parented by CronJobs) and ReplicaSets (parented by
// Deployments, StatefulSets, argo Rollouts, etc.). This list may change in the future, but
// is sufficient for now.
var indexedParents = map[string]indexedParent{
"Job": {Job, batchv1.SchemeGroupVersion.WithResource("jobs")},
"ReplicaSet": {RS, appsv1.SchemeGroupVersion.WithResource("replicasets")},
}

type indexedParent struct {
resource APIResource
gvr schema.GroupVersionResource
}
var indexedParents = map[string]APIResource{"Job": Job, "ReplicaSet": RS}

// GetRootOwnerKindAndName returns the resource's owner's type and metadata, using owner
// references from the Kubernetes API. Parent refs are recursively traversed to find the
Expand All @@ -310,10 +302,10 @@ func (api *MetadataAPI) GetRootOwnerKindAndName(ctx context.Context, tm *metav1.
log.Warnf("parent ref has no kind: %v", parentRef)
return tm, om
}
indexedResource, ok := indexedParents[parentRef.Kind]
resource, ok := indexedParents[parentRef.Kind]
if ok {
getFromInformer := func() (*metav1.TypeMeta, *metav1.ObjectMeta, error) {
parent, err := api.getByNamespace(indexedResource.resource, om.Namespace, parentRef.Name)
parent, err := api.getByNamespace(resource, om.Namespace, parentRef.Name)
if err != nil {
return nil, nil, err
}
Expand All @@ -328,20 +320,19 @@ func (api *MetadataAPI) GetRootOwnerKindAndName(ctx context.Context, tm *metav1.
if err == nil {
return parentTm, parentOm
}
log.Warnf("failed to retrieve %s from informer %s/%s: %s", parentRef.Kind, om.Namespace, parentRef.Name, err)
if retry {
parentFromApi, err := api.client.
Resource(indexedResource.gvr).
Namespace(om.Namespace).
Get(ctx, parentRef.Name, metav1.GetOptions{})
log.Warnf("failed to retrieve %s from informer %s/%s, resyncing informer cache", parentRef.Kind, om.Namespace, parentRef.Name)
// The proxy injnector runs very soon after a resource is created,
// which leaves a small possibility that the informer cache hasn't
// picked up the new resource yet. If that is the case, we force an
// informer resync to pick up the new resource.
api.Sync(nil)
Copy link
Member

@adleong adleong Nov 7, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This doesn't force an informer resync. This simply starts each informer (they are already started at this point) and waits for them to have their initial data. I believe that this is a no-op at this point and doesn't guarantee that the object we want will be in cache.

parentTm, parentOm, err := getFromInformer()
if err == nil {
if parentFromApi.ObjectMeta.Namespace == "" {
parentFromApi.ObjectMeta.Namespace = om.Namespace
}
return api.GetRootOwnerKindAndName(ctx, &parentFromApi.TypeMeta, &parentFromApi.ObjectMeta, retry)
return parentTm, parentOm
}
log.Warnf("failed to retrieve %s from direct API call %s/%s: %s", parentRef.Kind, om.Namespace, parentRef.Name, err)
}
log.Warnf("failed to retrieve %s from informer %s/%s: %s", parentRef.Kind, om.Namespace, parentRef.Name, err)
return &parentType, &metav1.ObjectMeta{Name: parentRef.Name, Namespace: om.Namespace}
}

Expand Down