better graph explorer

re-enable locations in dumps

tools/raptor_graph_explorer/tests/test_api.py

@@ -8,7 +8,10 @@ from raptor_graph_explorer.api import create_app
 
 def test_manifest_graph_motifs_and_pagination(built_output: Path):
     client=TestClient(create_app(built_output,max_page_size=2))
-    assert "Raptor Graph Explorer" in client.get("/").text
+    page = client.get("/")
+    assert "Raptor Graph Explorer" in page.text
+    assert page.headers["cache-control"] == "no-store"
+    assert client.get("/app.js").headers["cache-control"] == "no-store"
     assert "runLayout" in client.get("/app.js").text
     assert client.get("/api/manifest").status_code==200
     assert len(client.get("/api/reports").json())==2
@@ -43,3 +46,40 @@ def test_unsupported_database_schema_is_rejected(built_output: Path):
     db.execute("UPDATE metadata SET value='999' WHERE key='schema_version'");db.commit();db.close()
     with pytest.raises(ValueError,match="unsupported database schema"):
         create_app(built_output)
+
+
+def test_static_layout_and_label_settings(built_output: Path):
+    client = TestClient(create_app(built_output))
+    script = client.get("/app.js").text
+    assert 'id="metric"' not in client.get("/").text
+    assert "edgeSize" not in script and '$("metric")' not in script
+    assert "size: 1" in script
+    assert "localStorage" not in script
+    assert 'node.ssa_name || node.node_id' in script
+    assert 'node.ssa_summary.find(Boolean) || `op ${node.op_id}`' in script
+    motif_reducer = script[script.index("function reduceNode"):script.index("function reduceEdge")]
+    assert 'result.label = ""' not in motif_reducer
+    assert 'labelColor: {color: "#e7edf5"}' in script
+    assert "labelDensity: 0.6" in script
+    assert "labelGridCellSize: 120" in script
+    assert "labelRenderedSizeThreshold: 0" in script
+    assert "stagePadding: 60" in script
+    assert "forceLabel" not in script
+    assert 'getSetting("labelRenderer")' in script
+    assert "labelIntersectsNode" in script
+    assert "framedGraphToViewport" in script
+    assert "dx * dx + dy * dy < radius * radius" in script
+    assert "Math.min(6, 3 + Math.log2(1 + node.instance_count) / 8)" in script
+    assert 'getSetting("hoverRenderer")' in script
+    assert 'labelColor: {color: "#000000"}' in script
+    projection = script[script.index("async function rebuildProjection"):script.index("function expandOperation")]
+    assert projection.count("renderGraph()") == 1
+    assert '"Reset layout"' in script and '"Rerun layout"' in script
+    layout = script[script.index("async function runLayout"):script.index("function motifIdentity")]
+    assert '["operation", "raw"].includes' in layout
+    assert "new ELK().layout" in layout
+    assert '"elk.direction": "DOWN"' in layout
+    assert "fitMotif(motif)" in script
+    assert "setCustomBBox" in script
+    fit = script[script.index("function fitMotif"):script.index("function selectMotif")]
+    assert "setCustomBBox" in fit and "refresh()" in fit and "animatedReset()" in fit

tools/raptor_graph_explorer/tests/test_cli.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 
-from raptor_graph_explorer.cli import main
+from raptor_graph_explorer.cli import build_output, main
+from raptor_graph_explorer.database import connect_readonly
 
 
 def test_build_inspect_and_exit_codes(tmp_path: Path, fixture_dir: Path, capsys):
@@ -11,3 +12,31 @@ def test_build_inspect_and_exit_codes(tmp_path: Path, fixture_dir: Path, capsys)
     assert "demo_graph" in captured and "mappings:" in captured
     assert main(["build",str(fixture_dir),"--output",str(output)])==2
     assert main(["build",str(tmp_path/"missing"),"--output",str(tmp_path/"bad")])==2
+
+
+def test_multi_report_build_finalizes_readonly_database(tmp_path: Path):
+    reports = tmp_path / "reports"; reports.mkdir()
+    header = "Id,op_id,lane,core,ssa_name\n"
+    edge_header = "Source,Target,Weight,Type,stage,source_lane,target_lane,channel_id\n"
+    (reports / "a.nodes.csv").write_text(header + "gc:0,0,,,%0\n")
+    (reports / "a.edges.csv").write_text(edge_header)
+    (reports / "b.nodes.csv").write_text(header + "gc:0,0,,,%0\ngc:1,1,,,%1\n")
+    (reports / "b.edges.csv").write_text(
+        edge_header
+        + "gc:0,gc:1,1,tensor<1xf32>,b,,,\n"
+        + "gc:1,gc:missing,1,tensor<1xf32>,b,,,\n")
+
+    output = tmp_path / "output"
+    manifest = build_output([reports], output)
+    assert (output / "manifest.json").is_file()
+    assert [report["report_id"] for report in manifest["reports"]] == ["a", "b"]
+    assert manifest["diagnostic_count"] == 1
+
+    db = connect_readonly(output / "graph.sqlite")
+    counts = [tuple(row) for row in db.execute(
+        "SELECT report_id,raw_node_count,raw_edge_count,operation_node_count,operation_edge_count "
+        "FROM reports ORDER BY report_id")]
+    assert counts == [("a", 1, 0, 1, 0), ("b", 2, 1, 2, 1)]
+    assert tuple(db.execute("SELECT code,occurrence_count FROM diagnostics").fetchone()) == ("unknown_endpoint", 1)
+    assert db.execute("PRAGMA journal_mode").fetchone()[0] == "delete"
+    db.close()

tools/raptor_graph_explorer/tests/test_ingest.py

@@ -5,6 +5,14 @@ import sqlite3
 from pathlib import Path
 
 from raptor_graph_explorer.cli import build_output
+from raptor_graph_explorer.database import create_database
+from raptor_graph_explorer.ingest import _read_edges, _read_nodes
+from raptor_graph_explorer.schema import DiagnosticCollector, ReportPair
+
+
+def _temporary_tables(db: sqlite3.Connection) -> list[str]:
+    return [row[0] for row in db.execute(
+        "SELECT name FROM sqlite_temp_master WHERE type='table' ORDER BY name")]
 
 
 def test_normalization_unknown_columns_prefix_and_lane_fallback(built_output: Path):
@@ -56,3 +64,47 @@ def test_explicit_edge_lane_precedes_endpoint_lane(tmp_path: Path):
     assert db.execute("SELECT source_lane,target_lane FROM raw_edges").fetchone()==(9,8)
     assert db.execute("SELECT occurrence_count FROM diagnostics WHERE code='contradictory_lane'").fetchone()==(1,)
     db.close()
+
+
+def test_ingestion_drops_staging_tables_on_success_and_missing_columns(tmp_path: Path):
+    nodes = tmp_path / "good.nodes.csv"
+    nodes.write_text("Id,op_id,lane,core,ssa_name\ngc:0,0,,,%0\ngc:1,1,,,%1\n")
+    edges = tmp_path / "good.edges.csv"
+    edges.write_text(
+        "Source,Target,Weight,Type,stage,source_lane,target_lane,channel_id\n"
+        "gc:0,gc:1,1,tensor<1xf32>,good,,,\n")
+    pair = ReportPair("good", nodes, edges)
+    diagnostics = DiagnosticCollector()
+    db = create_database(tmp_path / "graph.sqlite")
+    db.execute("INSERT INTO reports(report_id,stage) VALUES('good','good')")
+
+    assert _read_nodes(db, pair, "good", diagnostics) == 2
+    assert _temporary_tables(db) == []
+    assert _read_edges(db, pair, "good", diagnostics) == 1
+    assert _temporary_tables(db) == []
+
+    missing_nodes = tmp_path / "missing.nodes.csv"
+    missing_nodes.write_text("Id,op_id,lane,core\ngc:2,2,,,\n")
+    db.execute("INSERT INTO reports(report_id,stage) VALUES('missing','missing')")
+    assert _read_nodes(db, ReportPair("missing", missing_nodes, edges), "missing", diagnostics) == 0
+    assert _temporary_tables(db) == []
+
+    missing_edges = tmp_path / "missing.edges.csv"
+    missing_edges.write_text("Source,Target\ngc:0,gc:1\n")
+    assert _read_edges(db, ReportPair("good", nodes, missing_edges), "good", diagnostics) == 0
+    assert _temporary_tables(db) == []
+    db.close()
+
+
+def test_required_columns_store_literal_empty_attributes(tmp_path: Path):
+    reports = tmp_path / "reports"; reports.mkdir()
+    (reports / "x.nodes.csv").write_text(
+        "Id,op_id,lane,core,ssa_name\ngc:0,0,,,%0\ngc:1,1,,,%1\n")
+    (reports / "x.edges.csv").write_text(
+        "Source,Target,Weight,Type,stage,source_lane,target_lane,channel_id\n"
+        "gc:0,gc:1,1,tensor<1xf32>,x,,,\n")
+    output = tmp_path / "out"; build_output([reports], output)
+    db = sqlite3.connect(output / "graph.sqlite")
+    assert db.execute("SELECT DISTINCT attributes_json FROM raw_nodes").fetchall() == [("{}",)]
+    assert db.execute("SELECT DISTINCT attributes_json FROM raw_edges").fetchall() == [("{}",)]
+    db.close()

tools/raptor_graph_explorer/tests/test_motifs.py

@@ -49,3 +49,9 @@ def test_external_entrance_rejected():
 def test_external_exit_rejected():
     edges=[("s","a"),("s","b"),("a","t"),("b","t"),("a","x")]
     assert detect_motifs(graph(edges))[0] == []
+
+
+def test_motifs_are_sorted_by_topological_rank_not_node_name():
+    edges = [("9", "a"), ("9", "b"), ("a", "z"), ("b", "z"), ("z", "10"),
+             ("10", "c"), ("10", "d"), ("c", "20"), ("d", "20")]
+    assert [motif.entry for motif in detect_motifs(graph(edges))[0]] == ["9", "10"]

tools/raptor_graph_explorer/tests/test_projection.py

@@ -6,8 +6,9 @@ from fastapi.testclient import TestClient
 import pytest
 
 from raptor_graph_explorer.api import create_app
+from raptor_graph_explorer.cli import build_output
 from raptor_graph_explorer.database import connect_readonly
-from raptor_graph_explorer.projection import DisplayGraphTooLarge, display_graph
+from raptor_graph_explorer.projection import DisplayGraphTooLarge, LANE_SPACING, display_graph
 
 
 def project(output: Path, report_id: str, expanded=(), *, view="operation", cap=5_000):
@@ -18,6 +19,36 @@ def project(output: Path, report_id: str, expanded=(), *, view="operation", cap=
         db.close()
 
 
+@pytest.fixture
+def sparse_projection_output(tmp_path: Path) -> Path:
+    reports = tmp_path / "reports"; reports.mkdir()
+    (reports / "sparse.nodes.csv").write_text(
+        "Id,op_id,lane,core,ssa_name\n"
+        "gc:0,0,,,%0\n"
+        "gcb:1:0:a,1,0,,%1\n"
+        "gcb:1:0:b,1,0,,%2\n"
+        "gcb:1:0:c,1,0,,%3\n"
+        "gcb:1:1000000,1,1000000,,%4\n"
+        "gc:1:scalar_a,1,,,%5\n"
+        "gc:1:scalar_b,1,,,%6\n"
+        "gc:2,2,,,%7\n")
+    edge_header = "Source,Target,Weight,Type,stage,source_lane,target_lane,channel_id\n"
+    pairs = [
+        ("gc:0", "gcb:1:0:a"),
+        ("gcb:1:0:a", "gcb:1:0:b"),
+        ("gcb:1:0:b", "gcb:1:0:c"),
+        ("gcb:1:0:c", "gcb:1:1000000"),
+        ("gcb:1:1000000", "gc:1:scalar_a"),
+        ("gc:1:scalar_a", "gc:1:scalar_b"),
+        ("gc:1:scalar_b", "gc:2"),
+    ]
+    (reports / "sparse.edges.csv").write_text(
+        edge_header + "".join(
+            f"{source},{target},1,tensor<1xf32>,sparse,,,\n" for source, target in pairs))
+    output = tmp_path / "output"; build_output([reports], output)
+    return output
+
+
 def test_spatial1_baseline_and_one_operation_expansion(regression_output: Path):
     db = connect_readonly(regression_output / "graph.sqlite")
     source_counts = db.execute("""
@@ -113,7 +144,7 @@ def test_display_api_and_static_raw_interactions(regression_output: Path):
     index = client.get("/").text
     script = client.get("/app.js").text
     assert '<option value="raw">Raw instances</option>' in index
-    assert 'report?.stage === "spatial4"' in script
+    assert '$("level").value = "operation"' in script
     assert 'local.representation === "raw"' in script
     reducer = script[script.index("function reduceEdge"):script.index("function applyFilters")]
     assert "state.graph.source(id)" not in reducer
@@ -123,3 +154,52 @@ def test_display_api_and_static_raw_interactions(regression_output: Path):
     assert "return;" in script[raw_edge_branch:aggregate_request]
     aggregate_edge = "operation:spatial1_graph:1->2"
     assert client.get(f"/api/aggregate-edges/{aggregate_edge}/matrix").status_code == 200
+
+
+def test_collapsed_sparse_lanes_have_compact_deterministic_extent(sparse_projection_output: Path):
+    first = project(sparse_projection_output, "sparse")
+    second = project(sparse_projection_output, "sparse")
+    aggregate_coordinates = [(node["x"], node["y"]) for node in first["nodes"]]
+    assert first["nodes"] == second["nodes"]
+    assert len(set(aggregate_coordinates)) == 3
+    assert max(y for _, y in aggregate_coordinates) - min(y for _, y in aggregate_coordinates) < 100 * LANE_SPACING
+
+
+def test_operation_edges_are_laid_out_top_to_bottom(sparse_projection_output: Path):
+    graph = project(sparse_projection_output, "sparse")
+    y = {node["op_id"]: node["y"] for node in graph["nodes"]}
+    cross_operation = [edge for edge in graph["edges"] if edge["source_op_id"] != edge["target_op_id"]]
+    assert cross_operation
+    assert all(y[edge["source_op_id"]] > y[edge["target_op_id"]] for edge in cross_operation)
+
+
+def test_expanded_sparse_lanes_use_compact_rows_and_valid_endpoints(sparse_projection_output: Path):
+    graph = project(sparse_projection_output, "sparse", [1])
+    nodes = {node["node_id"]: node for node in graph["nodes"] if node["representation"] == "raw"}
+    assert nodes["gcb:1:1000000"]["y"] - nodes["gcb:1:0:a"]["y"] == -LANE_SPACING
+    assert "operation:sparse:1" not in {node["display_id"] for node in graph["nodes"]}
+    displayed = {node["display_id"] for node in graph["nodes"]}
+    assert all(edge["source"] in displayed and edge["target"] in displayed for edge in graph["edges"])
+
+
+def test_same_lane_duplicates_are_centered_and_deterministic(sparse_projection_output: Path):
+    collapsed = project(sparse_projection_output, "sparse")
+    anchor_x = next(node["x"] for node in collapsed["nodes"] if node["op_id"] == 1)
+    first = project(sparse_projection_output, "sparse", [1])
+    second = project(sparse_projection_output, "sparse", [1])
+    duplicate_ids = ["gcb:1:0:a", "gcb:1:0:b", "gcb:1:0:c"]
+    nodes = {node["node_id"]: node for node in first["nodes"] if node["representation"] == "raw"}
+    xs = [nodes[node_id]["x"] for node_id in duplicate_ids]
+    assert len({nodes[node_id]["y"] for node_id in duplicate_ids}) == 1
+    assert len(set(xs)) == 3
+    assert sum(xs) / len(xs) == pytest.approx(anchor_x)
+    assert first["nodes"] == second["nodes"]
+
+
+def test_lane_less_nodes_follow_lane_rows_in_node_id_order(sparse_projection_output: Path):
+    graph = project(sparse_projection_output, "sparse", [1])
+    nodes = {node["node_id"]: node for node in graph["nodes"] if node["representation"] == "raw"}
+    lane_y = min(node["y"] for node in nodes.values() if node["lane"] is not None)
+    scalar_y = [nodes[node_id]["y"] for node_id in ("gc:1:scalar_a", "gc:1:scalar_b")]
+    assert scalar_y[0] < lane_y
+    assert scalar_y[1] - scalar_y[0] == -LANE_SPACING