I am done

This commit is contained in:
2024-10-30 22:14:35 +01:00
parent 720dc28c09
commit 40e2a747cf
36901 changed files with 5011519 additions and 0 deletions

View File

@ -0,0 +1,2 @@
from .find_first_use_of_broken_modules import find_first_use_of_broken_modules
from .trace_dependencies import trace_dependencies

View File

@ -0,0 +1,32 @@
from typing import Dict, List
from torch.package.package_exporter import PackagingError
__all__ = ["find_first_use_of_broken_modules"]
def find_first_use_of_broken_modules(exc: PackagingError) -> Dict[str, List[str]]:
"""
Find all broken modules in a PackagingError, and for each one, return the
dependency path in which the module was first encountered.
E.g. broken module m.n.o was added to a dependency graph while processing a.b.c,
then re-encountered while processing d.e.f. This method would return
{'m.n.o': ['a', 'b', 'c']}
Args:
exc: a PackagingError
Returns: A dict from broken module names to lists of module names in the path.
"""
assert isinstance(exc, PackagingError), "exception must be a PackagingError"
uses = {}
broken_module_names = [
m for m, attr in exc.dependency_graph.nodes.items() if attr.get("error", False)
]
for module_name in broken_module_names:
path = exc.dependency_graph.first_path(module_name)
uses[module_name] = path
return uses

View File

@ -0,0 +1,16 @@
from types import ModuleType
from typing import Any
from .._mangling import is_mangled
def is_from_package(obj: Any) -> bool:
"""
Return whether an object was loaded from a package.
Note: packaged objects from externed modules will return ``False``.
"""
if type(obj) == ModuleType:
return is_mangled(obj.__name__)
else:
return is_mangled(type(obj).__module__)

View File

@ -0,0 +1,64 @@
# mypy: allow-untyped-defs
import sys
from typing import Any, Callable, Iterable, List, Tuple
__all__ = ["trace_dependencies"]
def trace_dependencies(
callable: Callable[[Any], Any], inputs: Iterable[Tuple[Any, ...]]
) -> List[str]:
"""Trace the execution of a callable in order to determine which modules it uses.
Args:
callable: The callable to execute and trace.
inputs: The input to use during tracing. The modules used by 'callable' when invoked by each set of inputs
are union-ed to determine all modules used by the callable for the purpooses of packaging.
Returns: A list of the names of all modules used during callable execution.
"""
modules_used = set()
def record_used_modules(frame, event, arg):
# If the event being profiled is not a Python function
# call, there is nothing to do.
if event != "call":
return
# This is the name of the function that was called.
name = frame.f_code.co_name
module = None
# Try to determine the name of the module that the function
# is in:
# 1) Check the global namespace of the frame.
# 2) Check the local namespace of the frame.
# 3) To handle class instance method calls, check
# the attribute named 'name' of the object
# in the local namespace corresponding to "self".
if name in frame.f_globals:
module = frame.f_globals[name].__module__
elif name in frame.f_locals:
module = frame.f_locals[name].__module__
elif "self" in frame.f_locals:
method = getattr(frame.f_locals["self"], name, None)
module = method.__module__ if method else None
# If a module was found, add it to the set of used modules.
if module:
modules_used.add(module)
try:
# Attach record_used_modules as the profiler function.
sys.setprofile(record_used_modules)
# Execute the callable with all inputs.
for inp in inputs:
callable(*inp)
finally:
# Detach the profiler function.
sys.setprofile(None)
return list(modules_used)