diff --git a/importlib_resources/_common.py b/importlib_resources/_common.py
index 8df6b39..e95371c 100644
--- a/importlib_resources/_common.py
+++ b/importlib_resources/_common.py
@@ -93,12 +93,13 @@ def _infer_caller():
     """
 
     def is_this_file(frame_info):
-        return frame_info.filename == __file__
+        return frame_info.filename == stack[0].filename
 
     def is_wrapper(frame_info):
         return frame_info.function == 'wrapper'
 
-    not_this_file = itertools.filterfalse(is_this_file, inspect.stack())
+    stack = inspect.stack()
+    not_this_file = itertools.filterfalse(is_this_file, stack)
     # also exclude 'wrapper' due to singledispatch in the call stack
     callers = itertools.filterfalse(is_wrapper, not_this_file)
     return next(callers).frame
diff --git a/importlib_resources/tests/test_files.py b/importlib_resources/tests/test_files.py
index 564b2cc..9cb55d0 100644
--- a/importlib_resources/tests/test_files.py
+++ b/importlib_resources/tests/test_files.py
@@ -1,3 +1,7 @@
+import os
+import pathlib
+import py_compile
+import shutil
 import textwrap
 import unittest
 import warnings
@@ -7,6 +11,7 @@
 import importlib_resources as resources
 from ..abc import Traversable
 from . import util
+from .compat.py39 import os_helper, import_helper
 
 
 @contextlib.contextmanager
@@ -97,8 +102,8 @@ class ModuleFilesZipTests(DirectSpec, util.ZipSetup, ModulesFiles, unittest.Test
 
 class ImplicitContextFiles:
     set_val = textwrap.dedent(
-        """
-        import importlib_resources as res
+        f"""
+        import {resources.__name__} as res
         val = res.files().joinpath('res.txt').read_text(encoding='utf-8')
         """
     )
@@ -108,6 +113,10 @@ class ImplicitContextFiles:
             'submod.py': set_val,
             'res.txt': 'resources are the best',
         },
+        'frozenpkg': {
+            '__init__.py': set_val.replace(resources.__name__, 'c_resources'),
+            'res.txt': 'resources are the best',
+        },
     }
 
     def test_implicit_files_package(self):
@@ -122,6 +131,32 @@ def test_implicit_files_submodule(self):
         """
         assert importlib.import_module('somepkg.submod').val == 'resources are the best'
 
+    def _compile_importlib(self):
+        """
+        Make a compiled-only copy of the importlib resources package.
+        """
+        bin_site = self.fixtures.enter_context(os_helper.temp_dir())
+        c_resources = pathlib.Path(bin_site, 'c_resources')
+        sources = pathlib.Path(resources.__file__).parent
+        shutil.copytree(sources, c_resources, ignore=lambda *_: ['__pycache__'])
+
+        for dirpath, _, filenames in os.walk(c_resources):
+            for filename in filenames:
+                source_path = pathlib.Path(dirpath) / filename
+                cfile = source_path.with_suffix('.pyc')
+                py_compile.compile(source_path, cfile)
+                pathlib.Path.unlink(source_path)
+        self.fixtures.enter_context(import_helper.DirsOnSysPath(bin_site))
+
+    def test_implicit_files_with_compiled_importlib(self):
+        """
+        Caller detection works for compiled-only resources module.
+
+        python/cpython#123085
+        """
+        self._compile_importlib()
+        assert importlib.import_module('frozenpkg').val == 'resources are the best'
+
 
 class ImplicitContextFilesDiskTests(
     DirectSpec, util.DiskSetup, ImplicitContextFiles, unittest.TestCase
diff --git a/newsfragments/+0f77c990.bugfix.rst b/newsfragments/+0f77c990.bugfix.rst
new file mode 100644
index 0000000..99c0dfc
--- /dev/null
+++ b/newsfragments/+0f77c990.bugfix.rst
@@ -0,0 +1 @@
+When inferring the caller in ``files()`` correctly detect one's own module even when the resources package source is not present. (python/cpython#123085)
\ No newline at end of file