diff --git a/src/PIM/Common/Support/DebugDump.cpp b/src/PIM/Common/Support/DebugDump.cpp index 05b5f05..632a3a9 100644 --- a/src/PIM/Common/Support/DebugDump.cpp +++ b/src/PIM/Common/Support/DebugDump.cpp @@ -24,7 +24,7 @@ void dumpModule(mlir::ModuleOp moduleOp, const std::string& name, bool assumeVer llvm::raw_os_ostream os(file); mlir::OpPrintingFlags flags; - flags.elideLargeElementsAttrs().enableDebugInfo(false, false); + flags.elideLargeElementsAttrs().enableDebugInfo(true, false); if (assumeVerified) flags.assumeVerified(); moduleOp.print(os, flags); diff --git a/tools/raptor_graph_explorer/README.md b/tools/raptor_graph_explorer/README.md index abd3f21..9f98a96 100644 --- a/tools/raptor_graph_explorer/README.md +++ b/tools/raptor_graph_explorer/README.md @@ -155,11 +155,11 @@ Motifs are not inferred from rendered geometry. For each operation graph the too ## Viewer and API -Except for spatial4, the viewer initially fetches an aggregate operation graph unless the browser has a saved view choice. Sigma renders the graph with WebGL. Controls cover report/view/metric selection, text and tensor search, mapping filters, self edges, relayout, fitting, and motif selection. Raw nodes and edges have their own selection details and never call aggregate-only detail or mapping endpoints. Mapping panels remain available for operation aggregate edges. +The viewer initially fetches an aggregate operation graph, so a previously selected raw view cannot make a new report exceed the display safety cap before the first render. Browser responses disable caching so HTML and JavaScript from different tool revisions cannot be mixed. Sigma renders the graph with WebGL. Controls cover report/view selection, text and tensor search, mapping filters, self edges, relayout, fitting, and motif selection. Edges use a fixed width. Raw nodes and edges have their own selection details and never call aggregate-only detail or mapping endpoints. Nodes without an SSA name fall back to their source identity. Mapping panels remain available for operation aggregate edges. Operation expansion is a deterministic projection of the source graph. Expand selected, Expand all operations, Collapse selected operation, and Collapse all rebuild the complete display from the current expanded-operation set. An expanded operation's raw nodes replace its aggregate node, including isolated raw nodes. An aggregate edge is retained only when both endpoint operations are collapsed; otherwise its raw edges replace it with endpoints calculated from the complete expansion set. Expansion order therefore cannot leave stale endpoints or simultaneous aggregate/raw representations. -Operation ranks are calculated from the complete operation graph, including isolated operations. Collapsed nodes use stable operation anchors. Expanded nodes are sorted by lane and node ID; lane numbers form perpendicular rows, equal lanes align across adjacent operations, and lane-less nodes use distinct deterministic scalar rows. This same model is used by Rerun layout, so unrelated operations do not jump during expansion or collapse and disconnected nodes never pile up at `(0, 0)`. +Operation ranks are calculated from the complete operation graph, including isolated operations, and every view flows top-to-bottom. A collapsed operation reserves one displayed row. Expanded nodes use compact rows for distinct lanes in numeric order, followed by deterministic lane-less rows, so sparse lane numbers create no empty geometric space. Raw nodes sharing a lane receive centered deterministic horizontal offsets. Operation and raw views use stable server coordinates, and `Reset layout` restores them; core and node-kind views use ELK, and `Rerun layout` runs it again. Operation labels use the first SSA name; Sigma's collision grid thins normal labels, and a label is skipped when its measured screen rectangle intersects another visible node. Selected or hovered nodes keep the original white bubble with black text. Selecting a motif frames its member nodes on the first click. Aggregate-node sizes remain bounded. Browser dependencies are pinned in one place, `static/index.html`: @@ -192,8 +192,10 @@ Raw pages default to 100 and cannot exceed 500. Subgraph depth cannot exceed fiv ## Performance behavior - CSV readers stream rows and insert them in bounded batches. -- Temporary SQLite staging and bulk joins resolve endpoints; ingestion performs no per-edge node query. -- Secondary indexes are created after raw insertion. +- Temporary SQLite staging tables are function-scoped and dropped immediately; bulk joins resolve endpoints without per-edge node queries. +- CSVs without additional columns use a constant empty JSON representation. +- Ingestion closes its write connection before derived indexes, aggregation, motifs, and diagnostics reopen the database. +- Secondary indexes are created after raw insertion and focus on serving and aggregation queries. - Raw edges are never retained as a Python object graph or loaded into NetworkX. - Only aggregate operation nodes/edges enter NetworkX. - Mapping statistics are grouped in SQL; exact stencil comparison streams one operation pair. diff --git a/tools/raptor_graph_explorer/raptor_graph_explorer/api.py b/tools/raptor_graph_explorer/raptor_graph_explorer/api.py index 9ec47a8..c17e301 100644 --- a/tools/raptor_graph_explorer/raptor_graph_explorer/api.py +++ b/tools/raptor_graph_explorer/raptor_graph_explorer/api.py @@ -42,6 +42,12 @@ def create_app(output_directory: str | Path, *, max_page_size: int = 500, manifest = json.loads(manifest_path.read_text()) app = FastAPI(title="Raptor Graph Explorer", version=manifest.get("tool_version", "unknown")) + @app.middleware("http") + async def disable_browser_cache(request, call_next): + response = await call_next(request) + response.headers["Cache-Control"] = "no-store" + return response + def connection(): db = connect_readonly(database) try: diff --git a/tools/raptor_graph_explorer/raptor_graph_explorer/cli.py b/tools/raptor_graph_explorer/raptor_graph_explorer/cli.py index 1e4fbd4..922e46a 100644 --- a/tools/raptor_graph_explorer/raptor_graph_explorer/cli.py +++ b/tools/raptor_graph_explorer/raptor_graph_explorer/cli.py @@ -10,7 +10,7 @@ from pathlib import Path from . import __version__ from .aggregate import aggregate_report -from .database import connect_readonly, create_database, create_indexes, store_diagnostics +from .database import connect_readonly, connect_writable, create_database, create_indexes, store_diagnostics from .discovery import discover_reports from .ingest import ingest_report from .motifs import analyze_report @@ -52,6 +52,12 @@ def build_output(inputs: list[str | Path], output_directory: str | Path, *, stri with connection: for pair in pairs: ingest_report(connection, pair, diagnostics) + finally: + connection.close() + + connection = connect_writable(output / "graph.sqlite") + try: + with connection: create_indexes(connection) for pair in pairs: aggregate_report(connection, pair.report_id) diff --git a/tools/raptor_graph_explorer/raptor_graph_explorer/database.py b/tools/raptor_graph_explorer/raptor_graph_explorer/database.py index a2f387d..257798d 100644 --- a/tools/raptor_graph_explorer/raptor_graph_explorer/database.py +++ b/tools/raptor_graph_explorer/raptor_graph_explorer/database.py @@ -7,7 +7,6 @@ from pathlib import Path from .schema import Diagnostic, SCHEMA_VERSION DDL = """ -PRAGMA foreign_keys=ON; CREATE TABLE metadata(key TEXT PRIMARY KEY, value TEXT NOT NULL); CREATE TABLE reports( report_id TEXT PRIMARY KEY, stage TEXT NOT NULL, raw_node_count INTEGER NOT NULL DEFAULT 0, @@ -65,25 +64,25 @@ CREATE TABLE diagnostics( """ INDEX_DDL = """ -CREATE INDEX raw_nodes_report_op ON raw_nodes(report_id,op_id); CREATE INDEX raw_nodes_report_lane ON raw_nodes(report_id,op_id,lane); -CREATE INDEX raw_nodes_report_core ON raw_nodes(report_id,core); -CREATE INDEX raw_edges_source_node ON raw_edges(report_id,source_node_id); -CREATE INDEX raw_edges_target_node ON raw_edges(report_id,target_node_id); -CREATE INDEX raw_edges_source_op ON raw_edges(report_id,source_op_id,target_op_id); CREATE INDEX raw_edges_target_op ON raw_edges(report_id,target_op_id,source_op_id); CREATE INDEX raw_edges_mapping ON raw_edges(report_id,source_op_id,target_op_id,source_lane,target_lane); -CREATE INDEX raw_edges_channel ON raw_edges(report_id,channel_id); CREATE INDEX aggregate_nodes_lookup ON aggregate_nodes(report_id,level,op_id,core); CREATE INDEX aggregate_edges_lookup ON aggregate_edges(report_id,level); CREATE INDEX motifs_report ON motifs(report_id); """ -def create_database(path: Path) -> sqlite3.Connection: +def connect_writable(path: Path) -> sqlite3.Connection: connection = sqlite3.connect(path) connection.row_factory = sqlite3.Row - connection.executescript("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;" + DDL) + connection.executescript("PRAGMA foreign_keys=ON; PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;") + return connection + + +def create_database(path: Path) -> sqlite3.Connection: + connection = connect_writable(path) + connection.executescript(DDL) connection.execute("INSERT INTO metadata VALUES('schema_version',?)", (str(SCHEMA_VERSION),)) return connection diff --git a/tools/raptor_graph_explorer/raptor_graph_explorer/ingest.py b/tools/raptor_graph_explorer/raptor_graph_explorer/ingest.py index 9c6ced9..7417f03 100644 --- a/tools/raptor_graph_explorer/raptor_graph_explorer/ingest.py +++ b/tools/raptor_graph_explorer/raptor_graph_explorer/ingest.py @@ -4,6 +4,7 @@ import csv import json import re import sqlite3 +import sys from pathlib import Path from .schema import DiagnosticCollector, EDGE_COLUMNS, NODE_COLUMNS, NODE_KINDS, ReportPair @@ -36,9 +37,10 @@ def _id_parts(node_id: str) -> tuple[str, int | None, int | None]: return prefix, op_id, lane -def _extra(row: dict[str, str | None], known: set[str]) -> str: - return json.dumps({key: value for key, value in row.items() if key not in known}, - sort_keys=True, separators=(",", ":")) +def _extra(row: dict[str, str | None], columns: tuple[str, ...]) -> str: + if not columns: + return "{}" + return json.dumps({key: row.get(key) for key in columns}, sort_keys=True, separators=(",", ":")) def _batches(rows, size: int = BATCH_SIZE): @@ -61,6 +63,15 @@ def _check_columns(reader: csv.DictReader, required: set[str], kind: str, report return True +def _drop_staging(connection: sqlite3.Connection, table: str) -> None: + active_exception = sys.exc_info()[0] + try: + connection.execute(f"DROP TABLE IF EXISTS temp.{table}") + except sqlite3.Error: + if active_exception is None: + raise + + def _read_nodes(connection: sqlite3.Connection, pair: ReportPair, stage: str, diagnostics: DiagnosticCollector) -> int: connection.executescript(""" @@ -70,63 +81,67 @@ def _read_nodes(connection: sqlite3.Connection, pair: ReportPair, stage: str, core INTEGER, ssa_name TEXT, attributes_json TEXT ); """) - valid = 0 - with pair.nodes_path.open(newline="", encoding="utf-8-sig") as stream: - reader = csv.DictReader(stream) - if not _check_columns(reader, NODE_COLUMNS, "node", pair.report_id, diagnostics): - return 0 + try: + valid = 0 + with pair.nodes_path.open(newline="", encoding="utf-8-sig") as stream: + reader = csv.DictReader(stream) + if not _check_columns(reader, NODE_COLUMNS, "node", pair.report_id, diagnostics): + return 0 + extra_columns = tuple(column for column in reader.fieldnames or () if column not in NODE_COLUMNS) - def normalized(): - nonlocal valid - for row_number, row in enumerate(reader, 2): - node_id = (row.get("Id") or "").strip() - if not node_id: - diagnostics.add("invalid_node", "node Id is empty", pair.report_id, f"row {row_number}") - continue - prefix, id_op, id_lane = _id_parts(node_id) - try: - op_id = _integer(row.get("op_id"), "op_id") - lane = _integer(row.get("lane"), "lane") - core = _integer(row.get("core"), "core") - except ValueError as exc: - diagnostics.add("malformed_integer", str(exc), pair.report_id, f"row {row_number}") - continue - if op_id is None: - if id_op is None: - diagnostics.add("invalid_node", "missing op_id and unusable node ID fallback", pair.report_id, - f"row {row_number}: {node_id}") + def normalized(): + nonlocal valid + for row_number, row in enumerate(reader, 2): + node_id = (row.get("Id") or "").strip() + if not node_id: + diagnostics.add("invalid_node", "node Id is empty", pair.report_id, f"row {row_number}") continue - op_id = id_op - diagnostics.add("id_fallback", "op_id parsed from node ID", pair.report_id, node_id, - severity="warning") - if lane is None and prefix in {"gcb", "scb"} and id_lane is not None: - lane = id_lane - diagnostics.add("id_fallback", "lane parsed from node ID", pair.report_id, node_id, - severity="warning") - valid += 1 - yield (row_number, node_id, NODE_KINDS.get(prefix, f"unknown:{prefix}"), op_id, lane, core, - row.get("ssa_name") or "", _extra(row, NODE_COLUMNS)) + prefix, id_op, id_lane = _id_parts(node_id) + try: + op_id = _integer(row.get("op_id"), "op_id") + lane = _integer(row.get("lane"), "lane") + core = _integer(row.get("core"), "core") + except ValueError as exc: + diagnostics.add("malformed_integer", str(exc), pair.report_id, f"row {row_number}") + continue + if op_id is None: + if id_op is None: + diagnostics.add("invalid_node", "missing op_id and unusable node ID fallback", + pair.report_id, f"row {row_number}: {node_id}") + continue + op_id = id_op + diagnostics.add("id_fallback", "op_id parsed from node ID", pair.report_id, node_id, + severity="warning") + if lane is None and prefix in {"gcb", "scb"} and id_lane is not None: + lane = id_lane + diagnostics.add("id_fallback", "lane parsed from node ID", pair.report_id, node_id, + severity="warning") + valid += 1 + yield (row_number, node_id, NODE_KINDS.get(prefix, f"unknown:{prefix}"), op_id, lane, core, + row.get("ssa_name") or "", _extra(row, extra_columns)) - for batch in _batches(normalized()): - connection.executemany("INSERT INTO ingest_nodes VALUES(?,?,?,?,?,?,?,?)", batch) + for batch in _batches(normalized()): + connection.executemany("INSERT INTO ingest_nodes VALUES(?,?,?,?,?,?,?,?)", batch) - duplicate = connection.execute( - "SELECT COALESCE(SUM(c-1),0) FROM (SELECT COUNT(*) c FROM ingest_nodes GROUP BY node_id HAVING c>1)" - ).fetchone()[0] - if duplicate: - examples = [row[0] for row in connection.execute( - "SELECT node_id FROM ingest_nodes GROUP BY node_id HAVING COUNT(*)>1 ORDER BY node_id LIMIT 5")] - diagnostics.add("duplicate_node_id", "duplicate node ID; first row retained", pair.report_id, - ", ".join(examples), duplicate) - connection.execute(""" - INSERT INTO raw_nodes(report_id,stage,node_id,node_kind,op_id,lane,core,ssa_name,attributes_json) - SELECT ?,?,n.node_id,n.node_kind,n.op_id,n.lane,n.core,n.ssa_name,n.attributes_json - FROM ingest_nodes n - JOIN (SELECT node_id,MIN(row_number) first_row FROM ingest_nodes GROUP BY node_id) first - ON first.node_id=n.node_id AND first.first_row=n.row_number - ORDER BY n.row_number - """, (pair.report_id, stage)) - return valid - duplicate + duplicate = connection.execute( + "SELECT COALESCE(SUM(c-1),0) FROM (SELECT COUNT(*) c FROM ingest_nodes GROUP BY node_id HAVING c>1)" + ).fetchone()[0] + if duplicate: + examples = [row[0] for row in connection.execute( + "SELECT node_id FROM ingest_nodes GROUP BY node_id HAVING COUNT(*)>1 ORDER BY node_id LIMIT 5")] + diagnostics.add("duplicate_node_id", "duplicate node ID; first row retained", pair.report_id, + ", ".join(examples), duplicate) + connection.execute(""" + INSERT INTO raw_nodes(report_id,stage,node_id,node_kind,op_id,lane,core,ssa_name,attributes_json) + SELECT ?,?,n.node_id,n.node_kind,n.op_id,n.lane,n.core,n.ssa_name,n.attributes_json + FROM ingest_nodes n + JOIN (SELECT node_id,MIN(row_number) first_row FROM ingest_nodes GROUP BY node_id) first + ON first.node_id=n.node_id AND first.first_row=n.row_number + ORDER BY n.row_number + """, (pair.report_id, stage)) + return valid - duplicate + finally: + _drop_staging(connection, "ingest_nodes") def _read_edges(connection: sqlite3.Connection, pair: ReportPair, stage: str, @@ -139,74 +154,79 @@ def _read_edges(connection: sqlite3.Connection, pair: ReportPair, stage: str, tensor_type TEXT, channel_id INTEGER, attributes_json TEXT ); """) - with pair.edges_path.open(newline="", encoding="utf-8-sig") as stream: - reader = csv.DictReader(stream) - if not _check_columns(reader, EDGE_COLUMNS, "edge", pair.report_id, diagnostics): - return 0 + try: + with pair.edges_path.open(newline="", encoding="utf-8-sig") as stream: + reader = csv.DictReader(stream) + if not _check_columns(reader, EDGE_COLUMNS, "edge", pair.report_id, diagnostics): + return 0 + extra_columns = tuple(column for column in reader.fieldnames or () if column not in EDGE_COLUMNS) - def normalized(): - for row_number, row in enumerate(reader, 2): - source = (row.get("Source") or "").strip() - target = (row.get("Target") or "").strip() - if not source or not target: - diagnostics.add("invalid_edge", "edge Source or Target is empty", pair.report_id, - f"row {row_number}") - continue - try: - source_lane = _integer(row.get("source_lane"), "source_lane") - target_lane = _integer(row.get("target_lane"), "target_lane") - weight = _integer(row.get("Weight"), "Weight") - channel_id = _integer(row.get("channel_id"), "channel_id") - except ValueError as exc: - diagnostics.add("malformed_integer", str(exc), pair.report_id, f"row {row_number}") - continue - yield (row_number, (row.get("stage") or stage).strip() or stage, source, target, - source_lane, target_lane, weight, row.get("Type") or "", channel_id, - _extra(row, EDGE_COLUMNS)) + def normalized(): + for row_number, row in enumerate(reader, 2): + source = (row.get("Source") or "").strip() + target = (row.get("Target") or "").strip() + if not source or not target: + diagnostics.add("invalid_edge", "edge Source or Target is empty", pair.report_id, + f"row {row_number}") + continue + try: + source_lane = _integer(row.get("source_lane"), "source_lane") + target_lane = _integer(row.get("target_lane"), "target_lane") + weight = _integer(row.get("Weight"), "Weight") + channel_id = _integer(row.get("channel_id"), "channel_id") + except ValueError as exc: + diagnostics.add("malformed_integer", str(exc), pair.report_id, f"row {row_number}") + continue + yield (row_number, (row.get("stage") or stage).strip() or stage, source, target, + source_lane, target_lane, weight, row.get("Type") or "", channel_id, + _extra(row, extra_columns)) - for batch in _batches(normalized()): - connection.executemany("INSERT INTO ingest_edges VALUES(?,?,?,?,?,?,?,?,?,?)", batch) + for batch in _batches(normalized()): + connection.executemany("INSERT INTO ingest_edges VALUES(?,?,?,?,?,?,?,?,?,?)", batch) - unknown_count = connection.execute(""" - SELECT COUNT(*) FROM ingest_edges e - LEFT JOIN raw_nodes s ON s.report_id=? AND s.node_id=e.source_node_id - LEFT JOIN raw_nodes t ON t.report_id=? AND t.node_id=e.target_node_id - WHERE s.node_id IS NULL OR t.node_id IS NULL - """, (pair.report_id, pair.report_id)).fetchone()[0] - if unknown_count: - examples = [f"{row[0]}->{row[1]}" for row in connection.execute(""" - SELECT e.source_node_id,e.target_node_id FROM ingest_edges e + unknown_count = connection.execute(""" + SELECT COUNT(*) FROM ingest_edges e LEFT JOIN raw_nodes s ON s.report_id=? AND s.node_id=e.source_node_id LEFT JOIN raw_nodes t ON t.report_id=? AND t.node_id=e.target_node_id - WHERE s.node_id IS NULL OR t.node_id IS NULL ORDER BY e.row_number LIMIT 5 - """, (pair.report_id, pair.report_id))] - diagnostics.add("unknown_endpoint", "edge endpoint is absent from the matching report; edge omitted", - pair.report_id, ", ".join(examples), unknown_count) + WHERE s.node_id IS NULL OR t.node_id IS NULL + """, (pair.report_id, pair.report_id)).fetchone()[0] + if unknown_count: + examples = [f"{row[0]}->{row[1]}" for row in connection.execute(""" + SELECT e.source_node_id,e.target_node_id FROM ingest_edges e + LEFT JOIN raw_nodes s ON s.report_id=? AND s.node_id=e.source_node_id + LEFT JOIN raw_nodes t ON t.report_id=? AND t.node_id=e.target_node_id + WHERE s.node_id IS NULL OR t.node_id IS NULL ORDER BY e.row_number LIMIT 5 + """, (pair.report_id, pair.report_id))] + diagnostics.add("unknown_endpoint", "edge endpoint is absent from the matching report; edge omitted", + pair.report_id, ", ".join(examples), unknown_count) - contradictions = connection.execute(""" - SELECT COUNT(*) FROM ingest_edges e - JOIN raw_nodes s ON s.report_id=? AND s.node_id=e.source_node_id - JOIN raw_nodes t ON t.report_id=? AND t.node_id=e.target_node_id - WHERE (e.explicit_source_lane IS NOT NULL AND s.lane IS NOT NULL AND e.explicit_source_lane<>s.lane) - OR (e.explicit_target_lane IS NOT NULL AND t.lane IS NOT NULL AND e.explicit_target_lane<>t.lane) - """, (pair.report_id, pair.report_id)).fetchone()[0] - if contradictions: - diagnostics.add("contradictory_lane", "explicit edge lane contradicts endpoint lane; explicit lane retained", - pair.report_id, count=contradictions) + contradictions = connection.execute(""" + SELECT COUNT(*) FROM ingest_edges e + JOIN raw_nodes s ON s.report_id=? AND s.node_id=e.source_node_id + JOIN raw_nodes t ON t.report_id=? AND t.node_id=e.target_node_id + WHERE (e.explicit_source_lane IS NOT NULL AND s.lane IS NOT NULL AND e.explicit_source_lane<>s.lane) + OR (e.explicit_target_lane IS NOT NULL AND t.lane IS NOT NULL AND e.explicit_target_lane<>t.lane) + """, (pair.report_id, pair.report_id)).fetchone()[0] + if contradictions: + diagnostics.add("contradictory_lane", + "explicit edge lane contradicts endpoint lane; explicit lane retained", + pair.report_id, count=contradictions) - connection.execute(""" - INSERT INTO raw_edges( - edge_id,report_id,stage,source_node_id,target_node_id,source_op_id,target_op_id, - source_lane,target_lane,weight,tensor_type,channel_id,attributes_json) - SELECT printf('%s:e:%012d',?,e.row_number),?,e.stage,e.source_node_id,e.target_node_id,s.op_id,t.op_id, - COALESCE(e.explicit_source_lane,s.lane),COALESCE(e.explicit_target_lane,t.lane), - e.weight,e.tensor_type,e.channel_id,e.attributes_json - FROM ingest_edges e - JOIN raw_nodes s ON s.report_id=? AND s.node_id=e.source_node_id - JOIN raw_nodes t ON t.report_id=? AND t.node_id=e.target_node_id - ORDER BY e.row_number - """, (pair.report_id, pair.report_id, pair.report_id, pair.report_id)) - return connection.execute("SELECT COUNT(*) FROM raw_edges WHERE report_id=?", (pair.report_id,)).fetchone()[0] + connection.execute(""" + INSERT INTO raw_edges( + edge_id,report_id,stage,source_node_id,target_node_id,source_op_id,target_op_id, + source_lane,target_lane,weight,tensor_type,channel_id,attributes_json) + SELECT printf('%s:e:%012d',?,e.row_number),?,e.stage,e.source_node_id,e.target_node_id,s.op_id,t.op_id, + COALESCE(e.explicit_source_lane,s.lane),COALESCE(e.explicit_target_lane,t.lane), + e.weight,e.tensor_type,e.channel_id,e.attributes_json + FROM ingest_edges e + JOIN raw_nodes s ON s.report_id=? AND s.node_id=e.source_node_id + JOIN raw_nodes t ON t.report_id=? AND t.node_id=e.target_node_id + ORDER BY e.row_number + """, (pair.report_id, pair.report_id, pair.report_id, pair.report_id)) + return connection.execute("SELECT COUNT(*) FROM raw_edges WHERE report_id=?", (pair.report_id,)).fetchone()[0] + finally: + _drop_staging(connection, "ingest_edges") def ingest_report(connection: sqlite3.Connection, pair: ReportPair, diagnostics: DiagnosticCollector) -> None: diff --git a/tools/raptor_graph_explorer/raptor_graph_explorer/motifs.py b/tools/raptor_graph_explorer/raptor_graph_explorer/motifs.py index a83ba9d..bcf52e4 100644 --- a/tools/raptor_graph_explorer/raptor_graph_explorer/motifs.py +++ b/tools/raptor_graph_explorer/raptor_graph_explorer/motifs.py @@ -69,12 +69,15 @@ def _dominates(dominators: dict, entry: int, node: int) -> bool: def detect_motifs(graph: nx.DiGraph) -> tuple[list[Motif], int]: if not graph: return [], 0 - dag, components, _ = _condense(graph) + dag, components, member_to_component = _condense(graph) source, sink = "__source__", "__sink__" dag.add_node(source) dag.add_node(sink) dag.add_edges_from((source, node) for node in list(dag) if node not in {source, sink} and dag.in_degree(node) == 0) dag.add_edges_from((node, sink) for node in list(dag) if node not in {source, sink} and dag.out_degree(node) == 0) + ranks = {} + for component in nx.topological_sort(dag): + ranks[component] = max((ranks[parent] + 1 for parent in dag.predecessors(component)), default=0) dominators = nx.immediate_dominators(dag, source) post_dominators = nx.immediate_dominators(dag.reverse(copy=False), sink) motifs = [] @@ -113,7 +116,8 @@ def detect_motifs(graph: nx.DiGraph) -> tuple[list[Motif], int]: motifs.append(Motif(entry_aggregate, exit_aggregate, internal_aggregates, branch_aggregates, paths, len(expanded), subgraph.number_of_edges(), cycle, kind, 1.0 if kind != "split_rejoin" else 0.9)) - motifs.sort(key=lambda motif: (motif.entry, motif.exit, motif.internal)) + motifs.sort(key=lambda motif: (ranks[member_to_component[motif.entry]], motif.entry, + motif.exit, motif.internal)) return motifs, len(components) diff --git a/tools/raptor_graph_explorer/raptor_graph_explorer/projection.py b/tools/raptor_graph_explorer/raptor_graph_explorer/projection.py index 8ea4288..5aade7c 100644 --- a/tools/raptor_graph_explorer/raptor_graph_explorer/projection.py +++ b/tools/raptor_graph_explorer/raptor_graph_explorer/projection.py @@ -11,7 +11,9 @@ JSON_COLUMNS = { "tensor_type_histogram", } LANE_SPACING = 42 -OPERATION_SPACING = 240 +OPERATION_SPACING = 126 +RANK_SPACING = 600 +DUPLICATE_NODE_SPACING = 12 class DisplayGraphTooLarge(ValueError): @@ -25,7 +27,8 @@ def _record(row: sqlite3.Row) -> dict: return result -def _operation_anchors(nodes: list[dict], edges: list[dict]) -> dict[int, tuple[float, float]]: +def _operation_anchors(nodes: list[dict], edges: list[dict], + expanded_rows: dict[int, int]) -> dict[int, tuple[float, float]]: graph = nx.DiGraph() graph.add_nodes_from(node["op_id"] for node in nodes) graph.add_edges_from((edge["source_op_id"], edge["target_op_id"]) for edge in edges) @@ -39,17 +42,13 @@ def _operation_anchors(nodes: list[dict], edges: list[dict]) -> dict[int, tuple[ by_rank[operation_ranks[node["op_id"]]].append(node) anchors = {} + rank_offset = 0 for rank, ranked_nodes in sorted(by_rank.items()): ranked_nodes.sort(key=lambda item: item["op_id"]) - cursor = 0 - for node in ranked_nodes: - minimum = node["minimum_lane"] if node["minimum_lane"] is not None else 0 - maximum = node["maximum_lane"] if node["maximum_lane"] is not None else minimum - 1 - lane_rows = max(0, maximum - minimum + 1) - scalar_rows = max(0, node["instance_count"] - node["lane_count"]) - anchor_y = 0 if len(ranked_nodes) == 1 else cursor - minimum * LANE_SPACING - anchors[node["op_id"]] = (rank * OPERATION_SPACING, anchor_y) - cursor += max(1, lane_rows + scalar_rows) * LANE_SPACING + 2 * LANE_SPACING + for index, node in enumerate(ranked_nodes): + anchors[node["op_id"]] = (index * OPERATION_SPACING, -rank_offset) + rank_offset += max(expanded_rows.get(node["op_id"], 1) for node in ranked_nodes) * LANE_SPACING + rank_offset += RANK_SPACING return anchors @@ -90,6 +89,35 @@ def _raw_edges(db: sqlite3.Connection, report_id: str, expanded: set[int], all_e return [result[key] for key in sorted(result)] +def _display_rows(raw_nodes: list[dict]) -> tuple[ + dict[int, int], dict[int, dict[int, int]], dict[int, dict[str, int]], dict[str, float]]: + by_operation: dict[int, list[dict]] = defaultdict(list) + for node in raw_nodes: + by_operation[node["op_id"]].append(node) + + row_counts = {} + lane_rows = {} + scalar_rows = {} + duplicate_offsets = {} + for op_id, nodes in by_operation.items(): + lanes = sorted({node["lane"] for node in nodes if node["lane"] is not None}) + lane_rows[op_id] = {lane: index for index, lane in enumerate(lanes)} + scalars = sorted(node["node_id"] for node in nodes if node["lane"] is None) + scalar_rows[op_id] = {node_id: len(lanes) + index for index, node_id in enumerate(scalars)} + row_counts[op_id] = max(1, len(lanes) + len(scalars)) + + duplicates: dict[int, list[str]] = defaultdict(list) + for node in nodes: + if node["lane"] is not None: + duplicates[node["lane"]].append(node["node_id"]) + for node_ids in duplicates.values(): + node_ids.sort() + centre = (len(node_ids) - 1) / 2 + for index, node_id in enumerate(node_ids): + duplicate_offsets[node_id] = (index - centre) * DUPLICATE_NODE_SPACING + return row_counts, lane_rows, scalar_rows, duplicate_offsets + + def display_graph(db: sqlite3.Connection, report_id: str, *, view: str = "operation", expanded_operation_ids=(), expand_all: bool = False, cap: int = 5_000) -> dict: if view not in {"operation", "raw"}: @@ -119,7 +147,9 @@ def display_graph(db: sqlite3.Connection, report_id: str, *, view: str = "operat f"safety cap is {cap}. Use an aggregate view or increase the configured expansion cap." ) - anchors = _operation_anchors(operation_nodes, operation_edges) + raw_nodes = _raw_nodes(db, report_id, expanded, expanded == known) + row_counts, lane_rows, scalar_rows, duplicate_offsets = _display_rows(raw_nodes) + anchors = _operation_anchors(operation_nodes, operation_edges, row_counts) nodes = [] for node in operation_nodes: if node["op_id"] not in expanded: @@ -127,22 +157,15 @@ def display_graph(db: sqlite3.Connection, report_id: str, *, view: str = "operat nodes.append({**node, "display_id": node["aggregate_id"], "representation": "aggregate", "x": x, "y": y}) - raw_nodes = _raw_nodes(db, report_id, expanded, expanded == known) - lane_occurrences: dict[tuple[int, int], int] = defaultdict(int) - scalar_indices: dict[int, int] = defaultdict(int) - maximum_lanes = {node["op_id"]: node["maximum_lane"] for node in operation_nodes} for node in raw_nodes: anchor_x, anchor_y = anchors[node["op_id"]] if node["lane"] is None: - index = scalar_indices[node["op_id"]] - scalar_indices[node["op_id"]] += 1 - row = (maximum_lanes[node["op_id"]] if maximum_lanes[node["op_id"]] is not None else -1) + 1 + index - x, y = anchor_x, anchor_y + row * LANE_SPACING + row = scalar_rows[node["op_id"]][node["node_id"]] + x = anchor_x else: - key = (node["op_id"], node["lane"]) - x = anchor_x + lane_occurrences[key] * 12 - y = anchor_y + node["lane"] * LANE_SPACING - lane_occurrences[key] += 1 + row = lane_rows[node["op_id"]][node["lane"]] + x = anchor_x + duplicate_offsets[node["node_id"]] + y = anchor_y - row * LANE_SPACING nodes.append({**node, "display_id": node["node_id"], "representation": "raw", "x": x, "y": y}) edges = [] diff --git a/tools/raptor_graph_explorer/raptor_graph_explorer/static/app.js b/tools/raptor_graph_explorer/raptor_graph_explorer/static/app.js index 7616753..0bddd74 100644 --- a/tools/raptor_graph_explorer/raptor_graph_explorer/static/app.js +++ b/tools/raptor_graph_explorer/raptor_graph_explorer/static/app.js @@ -16,18 +16,16 @@ function definition(object, keys) { return `
${keys.filter(key => object[key] !== undefined).map(key => `
${html(key.replaceAll("_", " "))}
${html(typeof object[key] === "object" ? JSON.stringify(object[key]) : object[key])}
`).join("")}
`; } function status(message, error = false) { $("status").textContent = message; $("status").className = error ? "error" : ""; } -function selectedReport() { return state.reports.find(report => report.report_id === $("report").value); } -function defaultViewForReport(report) { - if (report?.stage === "spatial4") return "raw"; - const saved = localStorage.getItem("raptor-graph-view"); - return ["operation", "raw", "core", "node_kind"].includes(saved) ? saved : "operation"; +function updateLayoutButton() { + $("layout").textContent = ["operation", "raw"].includes($("level").value) ? "Reset layout" : "Rerun layout"; } async function loadReports() { state.reports = await api("/api/reports"); if (!state.reports.length) throw new Error("No reports were preprocessed."); $("report").innerHTML = state.reports.map(report => ``).join(""); - $("level").value = defaultViewForReport(selectedReport()); + $("level").value = "operation"; + updateLayoutButton(); await loadGraph(); } function aggregateGraph(data) { @@ -72,32 +70,60 @@ function updateStatus() { } else status(`${state.data.nodes.length} aggregate nodes, ${state.data.edges.length} aggregate edges`); } function nodeLabel(node) { - if (node.representation === "raw") return node.lane === null ? node.node_id : `${node.node_id} · lane ${node.lane}`; - return node.op_id !== null ? `op ${node.op_id}` : node.group_key; + if (node.representation === "raw") { + const name = node.ssa_name || node.node_id; + return node.lane === null ? name : `${name} · lane ${node.lane}`; + } + return node.op_id !== null ? node.ssa_summary.find(Boolean) || `op ${node.op_id}` : node.group_key; } -function edgeSize(edge) { - if (edge.representation === "raw") return 1; - const value = edge[$("metric").value] || 0; - return 1 + Math.log2(1 + Math.max(0, value)) / 3; +function labelIntersectsNode(context, data, settings) { + context.font = `${settings.labelWeight} ${settings.labelSize}px ${settings.labelFont}`; + const metrics = context.measureText(data.label), x = data.x + data.size + 3, y = data.y + settings.labelSize / 3; + const bounds = { + left: x - (metrics.actualBoundingBoxLeft || 0), right: x + (metrics.actualBoundingBoxRight || metrics.width), + top: y - (metrics.actualBoundingBoxAscent || settings.labelSize), bottom: y + (metrics.actualBoundingBoxDescent || 0), + }; + return state.graph.nodes().some(id => { + if (id === data.key) return false; + const node = state.renderer.getNodeDisplayData(id); + if (!node || node.hidden) return false; + const position = state.renderer.framedGraphToViewport(node), radius = state.renderer.scaleSize(node.size) + 1; + const dx = position.x - Math.max(bounds.left, Math.min(position.x, bounds.right)); + const dy = position.y - Math.max(bounds.top, Math.min(position.y, bounds.bottom)); + return dx * dx + dy * dy < radius * radius; + }); } async function renderGraph() { if (state.renderer) state.renderer.kill(); const graph = new graphology.MultiDirectedGraph(); for (const node of state.data.nodes) graph.addNode(node.display_id, { label: nodeLabel(node), x: node.x, y: node.y, - size: node.representation === "raw" ? 3 : 4 + Math.log2(1 + node.instance_count), + size: node.representation === "raw" ? 3 : Math.min(6, 3 + Math.log2(1 + node.instance_count) / 8), color: node.representation === "raw" ? colors.raw : colors[node.level], raw: node, searchable: JSON.stringify(node).toLowerCase(), }); for (const edge of state.data.edges) if (graph.hasNode(edge.source) && graph.hasNode(edge.target)) { graph.addDirectedEdgeWithKey(edge.display_id, edge.source, edge.target, { - size: edgeSize(edge), color: edge.representation === "raw" ? "#e0a94f" : "#718096", + size: 1, color: edge.representation === "raw" ? "#e0a94f" : "#718096", type: "arrow", raw: edge, }); } state.graph = graph; await runLayout(); - state.renderer = new Sigma(graph, $("graph"), {defaultEdgeType: "arrow", enableEdgeEvents: true, nodeReducer: reduceNode, edgeReducer: reduceEdge}); + state.renderer = new Sigma(graph, $("graph"), { + defaultEdgeType: "arrow", enableEdgeEvents: true, + labelColor: {color: "#e7edf5"}, labelWeight: "500", + labelDensity: 0.6, labelGridCellSize: 120, labelRenderedSizeThreshold: 0, + stagePadding: 60, + nodeReducer: reduceNode, edgeReducer: reduceEdge, + }); + const labelRenderer = state.renderer.getSetting("labelRenderer"); + state.renderer.setSetting("labelRenderer", (context, data, settings) => { + if (!labelIntersectsNode(context, data, settings)) labelRenderer(context, data, settings); + }); + const hoverRenderer = state.renderer.getSetting("hoverRenderer"); + state.renderer.setSetting("hoverRenderer", (context, data, settings) => + hoverRenderer(context, data, {...settings, labelColor: {color: "#000000"}})); state.renderer.on("clickNode", event => showNode(event.node)); state.renderer.on("clickEdge", event => showEdge(event.edge)); applyFilters(); @@ -111,7 +137,7 @@ async function runLayout() { } else { const layout = await new ELK().layout({ id: "root", - layoutOptions: {"elk.algorithm": "layered", "elk.direction": "RIGHT", "elk.spacing.nodeNode": "35", "elk.layered.spacing.nodeNodeBetweenLayers": "65"}, + layoutOptions: {"elk.algorithm": "layered", "elk.direction": "DOWN", "elk.spacing.nodeNode": "35", "elk.layered.spacing.nodeNodeBetweenLayers": "65"}, children: state.graph.nodes().map(id => ({id, width: 28, height: 28})), edges: state.graph.edges().map(id => ({id, sources: [state.graph.source(id)], targets: [state.graph.target(id)]})), }); @@ -128,7 +154,7 @@ function reduceNode(id, attributes) { if (state.motif) { const members = new Set([state.motif.entry_aggregate_id, state.motif.exit_aggregate_id, ...state.motif.internal_aggregate_ids]); const identity = motifIdentity(attributes.raw); - if (!members.has(identity)) { result.color = "#263341"; result.label = ""; } + if (!members.has(identity)) result.color = "#263341"; else if (identity === state.motif.entry_aggregate_id) { result.color = "#54d67b"; result.size *= 1.5; } else if (identity === state.motif.exit_aggregate_id) { result.color = "#ff6f6f"; result.size *= 1.5; } } @@ -210,11 +236,32 @@ async function loadMappingRows(id) { $("next").onclick = () => { state.mappingOffset += 25; loadMappingRows(id); }; } function renderMotifs() { - $("motifs").innerHTML = state.motifs.length ? state.motifs.map((motif, index) => ``).join("") : "No motifs detected."; + $("motifs").innerHTML = state.motifs.length ? state.motifs.map((motif, index) => ``).join("") : "No motifs detected."; document.querySelectorAll(".motif").forEach(button => button.onclick = () => selectMotif(state.motifs[button.dataset.index])); } +function operationName(aggregateId) { + const node = state.data.nodes.find(item => motifIdentity(item) === aggregateId); + const name = node?.ssa_summary?.[0] || node?.ssa_name; + return name || aggregateId; +} +function fitMotif(motif) { + const members = new Set([motif.entry_aggregate_id, motif.exit_aggregate_id, ...motif.internal_aggregate_ids]); + const positions = state.graph.nodes() + .filter(id => members.has(motifIdentity(state.graph.getNodeAttribute(id, "raw")))) + .map(id => state.graph.getNodeAttributes(id)); + if (!positions.length) return; + const xs = positions.map(node => node.x), ys = positions.map(node => node.y); + const width = Math.max(1, Math.max(...xs) - Math.min(...xs)); + const height = Math.max(1, Math.max(...ys) - Math.min(...ys)); + state.renderer.setCustomBBox({ + x: [Math.min(...xs) - width * 0.1, Math.max(...xs) + width * 0.1], + y: [Math.min(...ys) - height * 0.1, Math.max(...ys) + height * 0.1], + }); + state.renderer.refresh(); + state.renderer.getCamera().animatedReset(); +} function selectMotif(motif) { - state.motif = motif; applyFilters(); + state.motif = motif; applyFilters(); fitMotif(motif); $("details").innerHTML = '

Structural SCC-condensed split/rejoin; rendered geometry is not evidence.

' + definition(motif, ["motif_kind", "entry_aggregate_id", "exit_aggregate_id", "internal_aggregate_ids", "branch_successor_aggregate_ids", "representative_paths", "node_count", "edge_count", "contains_cycle", "confidence"]); } async function loadSummary() { @@ -223,10 +270,9 @@ async function loadSummary() { $("mapping").innerHTML = '' + Object.keys(summary.mapping_classes).map(kind => ``).join(""); } -$("report").onchange = () => { $("level").value = defaultViewForReport(selectedReport()); loadGraph(); }; -$("level").onchange = () => { localStorage.setItem("raptor-graph-view", $("level").value); loadGraph(); }; -$("metric").onchange = renderGraph; +$("report").onchange = () => { $("level").value = "operation"; updateLayoutButton(); loadGraph(); }; +$("level").onchange = () => { updateLayoutButton(); loadGraph(); }; for (const id of ["search", "mapping", "tensor", "selfEdges"]) $(id).addEventListener(id === "selfEdges" ? "change" : "input", applyFilters); -$("layout").onclick = runLayout; $("fit").onclick = () => state.renderer?.getCamera().animatedReset(); +$("layout").onclick = runLayout; $("fit").onclick = () => { state.renderer?.setCustomBBox(null); state.renderer?.refresh(); state.renderer?.getCamera().animatedReset(); }; $("expandAll").onclick = expandAllOperations; $("collapseAll").onclick = collapseAllOperations; loadReports().catch(error => status(error.message, true)); diff --git a/tools/raptor_graph_explorer/raptor_graph_explorer/static/index.html b/tools/raptor_graph_explorer/raptor_graph_explorer/static/index.html index ccd1409..76974aa 100644 --- a/tools/raptor_graph_explorer/raptor_graph_explorer/static/index.html +++ b/tools/raptor_graph_explorer/raptor_graph_explorer/static/index.html @@ -15,17 +15,16 @@
- + diff --git a/tools/raptor_graph_explorer/tests/test_api.py b/tools/raptor_graph_explorer/tests/test_api.py index 8f07441..d723b34 100644 --- a/tools/raptor_graph_explorer/tests/test_api.py +++ b/tools/raptor_graph_explorer/tests/test_api.py @@ -8,7 +8,10 @@ from raptor_graph_explorer.api import create_app def test_manifest_graph_motifs_and_pagination(built_output: Path): client=TestClient(create_app(built_output,max_page_size=2)) - assert "Raptor Graph Explorer" in client.get("/").text + page = client.get("/") + assert "Raptor Graph Explorer" in page.text + assert page.headers["cache-control"] == "no-store" + assert client.get("/app.js").headers["cache-control"] == "no-store" assert "runLayout" in client.get("/app.js").text assert client.get("/api/manifest").status_code==200 assert len(client.get("/api/reports").json())==2 @@ -43,3 +46,40 @@ def test_unsupported_database_schema_is_rejected(built_output: Path): db.execute("UPDATE metadata SET value='999' WHERE key='schema_version'");db.commit();db.close() with pytest.raises(ValueError,match="unsupported database schema"): create_app(built_output) + + +def test_static_layout_and_label_settings(built_output: Path): + client = TestClient(create_app(built_output)) + script = client.get("/app.js").text + assert 'id="metric"' not in client.get("/").text + assert "edgeSize" not in script and '$("metric")' not in script + assert "size: 1" in script + assert "localStorage" not in script + assert 'node.ssa_name || node.node_id' in script + assert 'node.ssa_summary.find(Boolean) || `op ${node.op_id}`' in script + motif_reducer = script[script.index("function reduceNode"):script.index("function reduceEdge")] + assert 'result.label = ""' not in motif_reducer + assert 'labelColor: {color: "#e7edf5"}' in script + assert "labelDensity: 0.6" in script + assert "labelGridCellSize: 120" in script + assert "labelRenderedSizeThreshold: 0" in script + assert "stagePadding: 60" in script + assert "forceLabel" not in script + assert 'getSetting("labelRenderer")' in script + assert "labelIntersectsNode" in script + assert "framedGraphToViewport" in script + assert "dx * dx + dy * dy < radius * radius" in script + assert "Math.min(6, 3 + Math.log2(1 + node.instance_count) / 8)" in script + assert 'getSetting("hoverRenderer")' in script + assert 'labelColor: {color: "#000000"}' in script + projection = script[script.index("async function rebuildProjection"):script.index("function expandOperation")] + assert projection.count("renderGraph()") == 1 + assert '"Reset layout"' in script and '"Rerun layout"' in script + layout = script[script.index("async function runLayout"):script.index("function motifIdentity")] + assert '["operation", "raw"].includes' in layout + assert "new ELK().layout" in layout + assert '"elk.direction": "DOWN"' in layout + assert "fitMotif(motif)" in script + assert "setCustomBBox" in script + fit = script[script.index("function fitMotif"):script.index("function selectMotif")] + assert "setCustomBBox" in fit and "refresh()" in fit and "animatedReset()" in fit diff --git a/tools/raptor_graph_explorer/tests/test_cli.py b/tools/raptor_graph_explorer/tests/test_cli.py index 13b399c..2ed4808 100644 --- a/tools/raptor_graph_explorer/tests/test_cli.py +++ b/tools/raptor_graph_explorer/tests/test_cli.py @@ -1,6 +1,7 @@ from pathlib import Path -from raptor_graph_explorer.cli import main +from raptor_graph_explorer.cli import build_output, main +from raptor_graph_explorer.database import connect_readonly def test_build_inspect_and_exit_codes(tmp_path: Path, fixture_dir: Path, capsys): @@ -11,3 +12,31 @@ def test_build_inspect_and_exit_codes(tmp_path: Path, fixture_dir: Path, capsys) assert "demo_graph" in captured and "mappings:" in captured assert main(["build",str(fixture_dir),"--output",str(output)])==2 assert main(["build",str(tmp_path/"missing"),"--output",str(tmp_path/"bad")])==2 + + +def test_multi_report_build_finalizes_readonly_database(tmp_path: Path): + reports = tmp_path / "reports"; reports.mkdir() + header = "Id,op_id,lane,core,ssa_name\n" + edge_header = "Source,Target,Weight,Type,stage,source_lane,target_lane,channel_id\n" + (reports / "a.nodes.csv").write_text(header + "gc:0,0,,,%0\n") + (reports / "a.edges.csv").write_text(edge_header) + (reports / "b.nodes.csv").write_text(header + "gc:0,0,,,%0\ngc:1,1,,,%1\n") + (reports / "b.edges.csv").write_text( + edge_header + + "gc:0,gc:1,1,tensor<1xf32>,b,,,\n" + + "gc:1,gc:missing,1,tensor<1xf32>,b,,,\n") + + output = tmp_path / "output" + manifest = build_output([reports], output) + assert (output / "manifest.json").is_file() + assert [report["report_id"] for report in manifest["reports"]] == ["a", "b"] + assert manifest["diagnostic_count"] == 1 + + db = connect_readonly(output / "graph.sqlite") + counts = [tuple(row) for row in db.execute( + "SELECT report_id,raw_node_count,raw_edge_count,operation_node_count,operation_edge_count " + "FROM reports ORDER BY report_id")] + assert counts == [("a", 1, 0, 1, 0), ("b", 2, 1, 2, 1)] + assert tuple(db.execute("SELECT code,occurrence_count FROM diagnostics").fetchone()) == ("unknown_endpoint", 1) + assert db.execute("PRAGMA journal_mode").fetchone()[0] == "delete" + db.close() diff --git a/tools/raptor_graph_explorer/tests/test_ingest.py b/tools/raptor_graph_explorer/tests/test_ingest.py index c00a30d..09647f6 100644 --- a/tools/raptor_graph_explorer/tests/test_ingest.py +++ b/tools/raptor_graph_explorer/tests/test_ingest.py @@ -5,6 +5,14 @@ import sqlite3 from pathlib import Path from raptor_graph_explorer.cli import build_output +from raptor_graph_explorer.database import create_database +from raptor_graph_explorer.ingest import _read_edges, _read_nodes +from raptor_graph_explorer.schema import DiagnosticCollector, ReportPair + + +def _temporary_tables(db: sqlite3.Connection) -> list[str]: + return [row[0] for row in db.execute( + "SELECT name FROM sqlite_temp_master WHERE type='table' ORDER BY name")] def test_normalization_unknown_columns_prefix_and_lane_fallback(built_output: Path): @@ -56,3 +64,47 @@ def test_explicit_edge_lane_precedes_endpoint_lane(tmp_path: Path): assert db.execute("SELECT source_lane,target_lane FROM raw_edges").fetchone()==(9,8) assert db.execute("SELECT occurrence_count FROM diagnostics WHERE code='contradictory_lane'").fetchone()==(1,) db.close() + + +def test_ingestion_drops_staging_tables_on_success_and_missing_columns(tmp_path: Path): + nodes = tmp_path / "good.nodes.csv" + nodes.write_text("Id,op_id,lane,core,ssa_name\ngc:0,0,,,%0\ngc:1,1,,,%1\n") + edges = tmp_path / "good.edges.csv" + edges.write_text( + "Source,Target,Weight,Type,stage,source_lane,target_lane,channel_id\n" + "gc:0,gc:1,1,tensor<1xf32>,good,,,\n") + pair = ReportPair("good", nodes, edges) + diagnostics = DiagnosticCollector() + db = create_database(tmp_path / "graph.sqlite") + db.execute("INSERT INTO reports(report_id,stage) VALUES('good','good')") + + assert _read_nodes(db, pair, "good", diagnostics) == 2 + assert _temporary_tables(db) == [] + assert _read_edges(db, pair, "good", diagnostics) == 1 + assert _temporary_tables(db) == [] + + missing_nodes = tmp_path / "missing.nodes.csv" + missing_nodes.write_text("Id,op_id,lane,core\ngc:2,2,,,\n") + db.execute("INSERT INTO reports(report_id,stage) VALUES('missing','missing')") + assert _read_nodes(db, ReportPair("missing", missing_nodes, edges), "missing", diagnostics) == 0 + assert _temporary_tables(db) == [] + + missing_edges = tmp_path / "missing.edges.csv" + missing_edges.write_text("Source,Target\ngc:0,gc:1\n") + assert _read_edges(db, ReportPair("good", nodes, missing_edges), "good", diagnostics) == 0 + assert _temporary_tables(db) == [] + db.close() + + +def test_required_columns_store_literal_empty_attributes(tmp_path: Path): + reports = tmp_path / "reports"; reports.mkdir() + (reports / "x.nodes.csv").write_text( + "Id,op_id,lane,core,ssa_name\ngc:0,0,,,%0\ngc:1,1,,,%1\n") + (reports / "x.edges.csv").write_text( + "Source,Target,Weight,Type,stage,source_lane,target_lane,channel_id\n" + "gc:0,gc:1,1,tensor<1xf32>,x,,,\n") + output = tmp_path / "out"; build_output([reports], output) + db = sqlite3.connect(output / "graph.sqlite") + assert db.execute("SELECT DISTINCT attributes_json FROM raw_nodes").fetchall() == [("{}",)] + assert db.execute("SELECT DISTINCT attributes_json FROM raw_edges").fetchall() == [("{}",)] + db.close() diff --git a/tools/raptor_graph_explorer/tests/test_motifs.py b/tools/raptor_graph_explorer/tests/test_motifs.py index 11a745b..6d03fd9 100644 --- a/tools/raptor_graph_explorer/tests/test_motifs.py +++ b/tools/raptor_graph_explorer/tests/test_motifs.py @@ -49,3 +49,9 @@ def test_external_entrance_rejected(): def test_external_exit_rejected(): edges=[("s","a"),("s","b"),("a","t"),("b","t"),("a","x")] assert detect_motifs(graph(edges))[0] == [] + + +def test_motifs_are_sorted_by_topological_rank_not_node_name(): + edges = [("9", "a"), ("9", "b"), ("a", "z"), ("b", "z"), ("z", "10"), + ("10", "c"), ("10", "d"), ("c", "20"), ("d", "20")] + assert [motif.entry for motif in detect_motifs(graph(edges))[0]] == ["9", "10"] diff --git a/tools/raptor_graph_explorer/tests/test_projection.py b/tools/raptor_graph_explorer/tests/test_projection.py index 16df5d4..51e6523 100644 --- a/tools/raptor_graph_explorer/tests/test_projection.py +++ b/tools/raptor_graph_explorer/tests/test_projection.py @@ -6,8 +6,9 @@ from fastapi.testclient import TestClient import pytest from raptor_graph_explorer.api import create_app +from raptor_graph_explorer.cli import build_output from raptor_graph_explorer.database import connect_readonly -from raptor_graph_explorer.projection import DisplayGraphTooLarge, display_graph +from raptor_graph_explorer.projection import DisplayGraphTooLarge, LANE_SPACING, display_graph def project(output: Path, report_id: str, expanded=(), *, view="operation", cap=5_000): @@ -18,6 +19,36 @@ def project(output: Path, report_id: str, expanded=(), *, view="operation", cap= db.close() +@pytest.fixture +def sparse_projection_output(tmp_path: Path) -> Path: + reports = tmp_path / "reports"; reports.mkdir() + (reports / "sparse.nodes.csv").write_text( + "Id,op_id,lane,core,ssa_name\n" + "gc:0,0,,,%0\n" + "gcb:1:0:a,1,0,,%1\n" + "gcb:1:0:b,1,0,,%2\n" + "gcb:1:0:c,1,0,,%3\n" + "gcb:1:1000000,1,1000000,,%4\n" + "gc:1:scalar_a,1,,,%5\n" + "gc:1:scalar_b,1,,,%6\n" + "gc:2,2,,,%7\n") + edge_header = "Source,Target,Weight,Type,stage,source_lane,target_lane,channel_id\n" + pairs = [ + ("gc:0", "gcb:1:0:a"), + ("gcb:1:0:a", "gcb:1:0:b"), + ("gcb:1:0:b", "gcb:1:0:c"), + ("gcb:1:0:c", "gcb:1:1000000"), + ("gcb:1:1000000", "gc:1:scalar_a"), + ("gc:1:scalar_a", "gc:1:scalar_b"), + ("gc:1:scalar_b", "gc:2"), + ] + (reports / "sparse.edges.csv").write_text( + edge_header + "".join( + f"{source},{target},1,tensor<1xf32>,sparse,,,\n" for source, target in pairs)) + output = tmp_path / "output"; build_output([reports], output) + return output + + def test_spatial1_baseline_and_one_operation_expansion(regression_output: Path): db = connect_readonly(regression_output / "graph.sqlite") source_counts = db.execute(""" @@ -113,7 +144,7 @@ def test_display_api_and_static_raw_interactions(regression_output: Path): index = client.get("/").text script = client.get("/app.js").text assert '' in index - assert 'report?.stage === "spatial4"' in script + assert '$("level").value = "operation"' in script assert 'local.representation === "raw"' in script reducer = script[script.index("function reduceEdge"):script.index("function applyFilters")] assert "state.graph.source(id)" not in reducer @@ -123,3 +154,52 @@ def test_display_api_and_static_raw_interactions(regression_output: Path): assert "return;" in script[raw_edge_branch:aggregate_request] aggregate_edge = "operation:spatial1_graph:1->2" assert client.get(f"/api/aggregate-edges/{aggregate_edge}/matrix").status_code == 200 + + +def test_collapsed_sparse_lanes_have_compact_deterministic_extent(sparse_projection_output: Path): + first = project(sparse_projection_output, "sparse") + second = project(sparse_projection_output, "sparse") + aggregate_coordinates = [(node["x"], node["y"]) for node in first["nodes"]] + assert first["nodes"] == second["nodes"] + assert len(set(aggregate_coordinates)) == 3 + assert max(y for _, y in aggregate_coordinates) - min(y for _, y in aggregate_coordinates) < 100 * LANE_SPACING + + +def test_operation_edges_are_laid_out_top_to_bottom(sparse_projection_output: Path): + graph = project(sparse_projection_output, "sparse") + y = {node["op_id"]: node["y"] for node in graph["nodes"]} + cross_operation = [edge for edge in graph["edges"] if edge["source_op_id"] != edge["target_op_id"]] + assert cross_operation + assert all(y[edge["source_op_id"]] > y[edge["target_op_id"]] for edge in cross_operation) + + +def test_expanded_sparse_lanes_use_compact_rows_and_valid_endpoints(sparse_projection_output: Path): + graph = project(sparse_projection_output, "sparse", [1]) + nodes = {node["node_id"]: node for node in graph["nodes"] if node["representation"] == "raw"} + assert nodes["gcb:1:1000000"]["y"] - nodes["gcb:1:0:a"]["y"] == -LANE_SPACING + assert "operation:sparse:1" not in {node["display_id"] for node in graph["nodes"]} + displayed = {node["display_id"] for node in graph["nodes"]} + assert all(edge["source"] in displayed and edge["target"] in displayed for edge in graph["edges"]) + + +def test_same_lane_duplicates_are_centered_and_deterministic(sparse_projection_output: Path): + collapsed = project(sparse_projection_output, "sparse") + anchor_x = next(node["x"] for node in collapsed["nodes"] if node["op_id"] == 1) + first = project(sparse_projection_output, "sparse", [1]) + second = project(sparse_projection_output, "sparse", [1]) + duplicate_ids = ["gcb:1:0:a", "gcb:1:0:b", "gcb:1:0:c"] + nodes = {node["node_id"]: node for node in first["nodes"] if node["representation"] == "raw"} + xs = [nodes[node_id]["x"] for node_id in duplicate_ids] + assert len({nodes[node_id]["y"] for node_id in duplicate_ids}) == 1 + assert len(set(xs)) == 3 + assert sum(xs) / len(xs) == pytest.approx(anchor_x) + assert first["nodes"] == second["nodes"] + + +def test_lane_less_nodes_follow_lane_rows_in_node_id_order(sparse_projection_output: Path): + graph = project(sparse_projection_output, "sparse", [1]) + nodes = {node["node_id"]: node for node in graph["nodes"] if node["representation"] == "raw"} + lane_y = min(node["y"] for node in nodes.values() if node["lane"] is not None) + scalar_y = [nodes[node_id]["y"] for node_id in ("gc:1:scalar_a", "gc:1:scalar_b")] + assert scalar_y[0] < lane_y + assert scalar_y[1] - scalar_y[0] == -LANE_SPACING