293 lines
11 KiB
Python
293 lines
11 KiB
Python
#!/usr/bin/env python
|
|
|
|
# Copyright (c) ONNX Project Contributors
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from typing import IO, Any, Sequence
|
|
|
|
from onnx import AttributeProto, defs, load
|
|
from onnx.backend.test.case import collect_snippets
|
|
from onnx.backend.test.loader import load_model_tests
|
|
from onnx.backend.test.runner import Runner
|
|
|
|
|
|
def is_ml(schemas: Sequence[defs.OpSchema]) -> bool:
|
|
return any(s.domain == "ai.onnx.ml" for s in schemas)
|
|
|
|
|
|
def gen_outlines(f: IO[Any], ml: bool) -> None:
|
|
f.write("# Test Coverage Report")
|
|
if ml:
|
|
f.write(" (ONNX-ML Operators)\n")
|
|
else:
|
|
f.write(" (ONNX Core Operators)\n")
|
|
f.write("## Outlines\n")
|
|
f.write("* [Node Test Coverage](#node-test-coverage)\n")
|
|
f.write("* [Model Test Coverage](#model-test-coverage)\n")
|
|
f.write("* [Overall Test Coverage](#overall-test-coverage)\n")
|
|
|
|
|
|
common_covered: Sequence[str] = []
|
|
experimental_covered: Sequence[str] = []
|
|
|
|
|
|
def gen_node_test_coverage(
|
|
schemas: Sequence[defs.OpSchema], f: IO[Any], ml: bool
|
|
) -> None:
|
|
global common_covered # noqa: PLW0603
|
|
global experimental_covered # noqa: PLW0603
|
|
generators = set(
|
|
{
|
|
"Multinomial",
|
|
"RandomNormal",
|
|
"RandomNormalLike",
|
|
"RandomUniform",
|
|
"RandomUniformLike",
|
|
}
|
|
)
|
|
node_tests = collect_snippets()
|
|
common_covered = sorted(
|
|
s.name
|
|
for s in schemas
|
|
if s.name in node_tests
|
|
and s.support_level == defs.OpSchema.SupportType.COMMON
|
|
and (s.domain == "ai.onnx.ml") == ml
|
|
)
|
|
common_no_cover = sorted(
|
|
s.name
|
|
for s in schemas
|
|
if s.name not in node_tests
|
|
and s.support_level == defs.OpSchema.SupportType.COMMON
|
|
and (s.domain == "ai.onnx.ml") == ml
|
|
)
|
|
common_generator = sorted(name for name in common_no_cover if name in generators)
|
|
experimental_covered = sorted(
|
|
s.name
|
|
for s in schemas
|
|
if s.name in node_tests
|
|
and s.support_level == defs.OpSchema.SupportType.EXPERIMENTAL
|
|
and (s.domain == "ai.onnx.ml") == ml
|
|
)
|
|
experimental_no_cover = sorted(
|
|
s.name
|
|
for s in schemas
|
|
if s.name not in node_tests
|
|
and s.support_level == defs.OpSchema.SupportType.EXPERIMENTAL
|
|
and (s.domain == "ai.onnx.ml") == ml
|
|
)
|
|
experimental_generator = sorted(
|
|
name for name in experimental_no_cover if name in generators
|
|
)
|
|
num_common = len(common_covered) + len(common_no_cover) - len(common_generator)
|
|
num_experimental = (
|
|
len(experimental_covered)
|
|
+ len(experimental_no_cover)
|
|
- len(experimental_generator)
|
|
)
|
|
f.write("# Node Test Coverage\n")
|
|
f.write("## Summary\n")
|
|
if num_common:
|
|
f.write(
|
|
f"Node tests have covered {len(common_covered)}/{num_common} "
|
|
f"({len(common_covered) / float(num_common) * 100:.2f}%, {len(common_generator)} "
|
|
f"generators excluded) common operators.\n\n"
|
|
)
|
|
else:
|
|
f.write("Node tests have covered 0/0 (N/A) common operators. \n\n")
|
|
if num_experimental:
|
|
f.write(
|
|
"Node tests have covered {}/{} ({:.2f}%, {} generators excluded) " # noqa: UP032
|
|
"experimental operators.\n\n".format(
|
|
len(experimental_covered),
|
|
num_experimental,
|
|
(len(experimental_covered) / float(num_experimental) * 100),
|
|
len(experimental_generator),
|
|
)
|
|
)
|
|
else:
|
|
f.write("Node tests have covered 0/0 (N/A) experimental operators.\n\n")
|
|
titles = [
|
|
"💚Covered Common Operators",
|
|
"💔No Cover Common Operators",
|
|
"💚Covered Experimental Operators",
|
|
"💔No Cover Experimental Operators",
|
|
]
|
|
all_lists = [
|
|
common_covered,
|
|
common_no_cover,
|
|
experimental_covered,
|
|
experimental_no_cover,
|
|
]
|
|
for t in titles:
|
|
f.write(f"* [{t[9:]}](#{t[9:].lower().replace(' ', '-')})\n")
|
|
f.write("\n")
|
|
for t, l in zip(titles, all_lists): # noqa: E741
|
|
f.write(f"## {t}\n")
|
|
for s in l:
|
|
f.write(f"### {s}")
|
|
if s in node_tests:
|
|
f.write(
|
|
f"\nThere are {len(node_tests[s])} test cases, listed as following:\n"
|
|
)
|
|
for summary, code in sorted(node_tests[s]):
|
|
f.write("<details>\n")
|
|
f.write(f"<summary>{summary}</summary>\n\n")
|
|
f.write(f"```python\n{code}\n```\n\n")
|
|
f.write("</details>\n")
|
|
else: # noqa: PLR5501
|
|
if s in generators:
|
|
f.write(" (random generator operator)\n")
|
|
else:
|
|
f.write(" (call for test cases)\n")
|
|
f.write("\n\n")
|
|
f.write("<br/>\n\n")
|
|
|
|
|
|
def gen_model_test_coverage(
|
|
schemas: Sequence[defs.OpSchema], f: IO[Any], ml: bool
|
|
) -> None:
|
|
f.write("# Model Test Coverage\n")
|
|
# Process schemas
|
|
schema_dict = {}
|
|
for schema in schemas:
|
|
schema_dict[schema.name] = schema
|
|
# Load models from each model test using Runner.prepare_model_data
|
|
# Need to grab associated nodes
|
|
attrs: dict[str, dict[str, list[Any]]] = {}
|
|
model_paths: list[Any] = []
|
|
for rt in load_model_tests(kind="real"):
|
|
if rt.url.startswith("onnx/backend/test/data/light/"):
|
|
# testing local files
|
|
model_name = os.path.normpath(
|
|
os.path.join(os.path.dirname(__file__), "..", "..", "..", rt.url)
|
|
)
|
|
if not os.path.exists(model_name):
|
|
raise FileNotFoundError(f"Unable to find model {model_name!r}.")
|
|
model_paths.append(model_name)
|
|
else:
|
|
model_dir = Runner.prepare_model_data(rt)
|
|
model_paths.append(os.path.join(model_dir, "model.onnx"))
|
|
model_paths.sort()
|
|
model_written = False
|
|
for model_pb_path in model_paths:
|
|
model = load(model_pb_path)
|
|
if ml:
|
|
ml_present = False
|
|
for opset in model.opset_import:
|
|
if opset.domain == "ai.onnx.ml":
|
|
ml_present = True
|
|
if not ml_present:
|
|
continue
|
|
else:
|
|
model_written = True
|
|
f.write(f"## {model.graph.name}\n")
|
|
# Deconstruct model
|
|
num_covered = 0
|
|
for node in model.graph.node:
|
|
if node.op_type in common_covered or node.op_type in experimental_covered:
|
|
num_covered += 1
|
|
# Add details of which nodes are/aren't covered
|
|
# Iterate through and store each node's attributes
|
|
for attr in node.attribute:
|
|
if node.op_type not in attrs:
|
|
attrs[node.op_type] = {}
|
|
if attr.name not in attrs[node.op_type]:
|
|
attrs[node.op_type][attr.name] = []
|
|
if attr.type == AttributeProto.FLOAT:
|
|
if attr.f not in attrs[node.op_type][attr.name]:
|
|
attrs[node.op_type][attr.name].append(attr.f)
|
|
elif attr.type == AttributeProto.INT:
|
|
if attr.i not in attrs[node.op_type][attr.name]:
|
|
attrs[node.op_type][attr.name].append(attr.i)
|
|
elif attr.type == AttributeProto.STRING:
|
|
if attr.s not in attrs[node.op_type][attr.name]:
|
|
attrs[node.op_type][attr.name].append(attr.s)
|
|
elif attr.type == AttributeProto.TENSOR:
|
|
if attr.t not in attrs[node.op_type][attr.name]:
|
|
attrs[node.op_type][attr.name].append(attr.t)
|
|
elif attr.type == AttributeProto.GRAPH:
|
|
if attr.g not in attrs[node.op_type][attr.name]:
|
|
attrs[node.op_type][attr.name].append(attr.g)
|
|
elif attr.type == AttributeProto.FLOATS:
|
|
if attr.floats not in attrs[node.op_type][attr.name]:
|
|
attrs[node.op_type][attr.name].append(attr.floats)
|
|
elif attr.type == AttributeProto.INTS:
|
|
if attr.ints not in attrs[node.op_type][attr.name]:
|
|
attrs[node.op_type][attr.name].append(attr.ints)
|
|
elif attr.type == AttributeProto.STRINGS:
|
|
if attr.strings not in attrs[node.op_type][attr.name]:
|
|
attrs[node.op_type][attr.name].append(attr.strings)
|
|
elif attr.type == AttributeProto.TENSORS:
|
|
if attr.tensors not in attrs[node.op_type][attr.name]:
|
|
attrs[node.op_type][attr.name].append(attr.tensors)
|
|
elif attr.type == AttributeProto.GRAPHS:
|
|
if attr.graphs not in attrs[node.op_type][attr.name]:
|
|
attrs[node.op_type][attr.name].append(attr.graphs)
|
|
f.write(
|
|
f"\n{model.graph.name} has {num_covered} nodes. "
|
|
f"Of these, {len(model.graph.node)} are covered by node tests "
|
|
f"({100.0 * float(num_covered) / float(len(model.graph.node))}%)\n\n\n"
|
|
)
|
|
# Iterate through attrs, print
|
|
f.write("<details>\n")
|
|
f.write("<summary>nodes</summary>\n\n")
|
|
for op in sorted(attrs):
|
|
f.write("<details>\n")
|
|
# Get total number of attributes for node schema
|
|
f.write(
|
|
f"<summary>{op}: {len(attrs[op])} out of {len(schema_dict[op].attributes)} attributes covered</summary>\n\n"
|
|
)
|
|
for attribute in sorted(schema_dict[op].attributes):
|
|
if attribute in attrs[op]:
|
|
f.write(f"{attribute}: {len(attrs[op][attribute])}\n")
|
|
else:
|
|
f.write(f"{attribute}: 0\n")
|
|
f.write("</details>\n")
|
|
f.write("</details>\n\n\n")
|
|
if not model_written and ml:
|
|
f.write("No model tests present for selected domain\n")
|
|
|
|
|
|
def gen_overall_test_coverage(
|
|
schemas: Sequence[defs.OpSchema], f: IO[Any], ml: bool # noqa: ARG001
|
|
) -> None:
|
|
f.write("# Overall Test Coverage\n")
|
|
f.write("## To be filled.\n")
|
|
|
|
|
|
def gen_spdx(f: IO[Any]) -> None:
|
|
f.write("<!--- SPDX-License-Identifier: Apache-2.0 -->\n")
|
|
|
|
|
|
def main() -> None:
|
|
base_dir = os.path.dirname(
|
|
os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
|
|
)
|
|
docs_dir = os.path.join(base_dir, "docs")
|
|
schemas = defs.get_all_schemas()
|
|
|
|
has_ml = is_ml(schemas)
|
|
fname = os.path.join(docs_dir, "TestCoverage.md")
|
|
with open(fname, "w+", newline="", encoding="utf-8") as f: # type: ignore
|
|
gen_spdx(f)
|
|
gen_outlines(f, False)
|
|
gen_node_test_coverage(schemas, f, False)
|
|
gen_model_test_coverage(schemas, f, False)
|
|
gen_overall_test_coverage(schemas, f, False)
|
|
|
|
if has_ml:
|
|
fname = os.path.join(docs_dir, "TestCoverage-ml.md")
|
|
with open(fname, "w+", newline="", encoding="utf-8") as f: # type: ignore
|
|
gen_spdx(f)
|
|
gen_outlines(f, True)
|
|
gen_node_test_coverage(schemas, f, True)
|
|
gen_model_test_coverage(schemas, f, True)
|
|
gen_overall_test_coverage(schemas, f, True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|