Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions backend/tests/app/graph/test_router.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
DataProductDatasetAssociationFactory,
DataProductFactory,
DatasetFactory,
DomainFactory,
TechnicalAssetFactory,
)

Expand Down Expand Up @@ -67,3 +68,38 @@ def test_data_products_only_arrow_points_producer_to_consumer(self, client):
assert edge["target"] == str(consumer.id), (
"Edge target should be the consumer (dataset reader)"
)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am unsure what the tests add to be honest, the logic did not change? So unsure why they are required. Can you explain to me the reasoning for adding these?

The includes domain fields tests for example could just be the last 2 asserts that are added to test_get_graph_data.

def test_graph_nodes_include_domain_fields(self, client):
domain = DomainFactory()
DataProductFactory(domain=domain)
response = client.get(ENDPOINT, params={"output_port_nodes_enabled": "false"})
assert response.status_code == 200, response.text
node = next(
n for n in response.json()["nodes"] if n["type"] == "dataProductNode"
)
assert node["data"]["domain_id"] == str(domain.id)
assert node["data"]["domain"] == domain.name

def test_output_port_inherits_domain_from_parent_data_product(self, client):
domain = DomainFactory()
data_product = DataProductFactory(domain=domain)
DatasetFactory(data_product=data_product)
response = client.get(ENDPOINT)
assert response.status_code == 200, response.text
dataset_node = next(
n for n in response.json()["nodes"] if n["type"] == "datasetNode"
)
assert dataset_node["data"]["domain_id"] == str(domain.id)
assert dataset_node["data"]["domain"] == domain.name

def test_graph_nodes_from_different_domains(self, client):
domain_a = DomainFactory()
domain_b = DomainFactory()
DataProductFactory(domain=domain_a)
DataProductFactory(domain=domain_b)
response = client.get(ENDPOINT, params={"output_port_nodes_enabled": "false"})
assert response.status_code == 200, response.text
nodes = response.json()["nodes"]
assert len(nodes) == 2
domain_ids = {n["data"]["domain_id"] for n in nodes}
assert domain_ids == {str(domain_a.id), str(domain_b.id)}
2 changes: 2 additions & 0 deletions docs/docs/release-notes.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,15 @@ sidebar_position: 200

### features

- **[Explorer]**: Domain container nodes are now visible in the global graph explorer, grouping data products by domain with distinct colours
- **[Postgresql]**: Postgresql plugin defining technical assets

### bugfixes

- **[Technical Asset]**: Radio button is selected but not captured causing validation issues
- **[Explorer]**: Fixed reversed arrow direction in the "Data Products" only view. Arrows now point from Producer to Consumer, consistent with the "All" view.


## 0.5.1

### features
Expand Down
1 change: 1 addition & 0 deletions frontend/public/locales/en/translation.json
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@
"Technical Assets": "Technical Assets",
"Undefined group": "Undefined group",
"All": "All",
"Show Domains": "Show Domains",
"Select a node": "Select a node",
"Search event history": "Search event history",
"Showing {{range0}}-{{range1}} of {{count}} history items_one": "Showing {{range0}}-{{range1}} of {{count}} history item",
Expand Down
56 changes: 27 additions & 29 deletions frontend/src/components/charts/node-editor/node-editor.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -44,34 +44,32 @@ export function NodeEditor({
return <LoadingSpinner />;
}
return (
<>
<ReactFlow
nodes={nodes}
edges={edges}
onConnect={onConnect}
onNodesChange={onNodesChange}
onEdgesChange={onEdgesChange}
onNodeClick={onNodeClick}
onPaneClick={onPaneClick}
fitView
onInit={(instance) => instance.fitView(defaultFitViewOptions)}
minZoom={MIN_ZOOM}
maxZoom={MAX_ZOOM}
zoomOnPinch
zoomOnDoubleClick
connectionLineType={ConnectionLineType.SmoothStep}
fitViewOptions={defaultFitViewOptions}
className={styles.nodeEditor}
nodeTypes={nodeTypes}
edgeTypes={edgeTypes}
elevateNodesOnSelect
nodesFocusable
attributionPosition={'bottom-left'}
{...editorProps}
>
<Background />
<Controls position={'top-right'} showInteractive={false} fitViewOptions={defaultFitViewOptions} />
</ReactFlow>
</>
<ReactFlow
nodes={nodes}
edges={edges}
onConnect={onConnect}
onNodesChange={onNodesChange}
onEdgesChange={onEdgesChange}
onNodeClick={onNodeClick}
onPaneClick={onPaneClick}
fitView
onInit={(instance) => instance.fitView(defaultFitViewOptions)}
minZoom={MIN_ZOOM}
maxZoom={MAX_ZOOM}
zoomOnPinch
zoomOnDoubleClick
connectionLineType={ConnectionLineType.SmoothStep}
fitViewOptions={defaultFitViewOptions}
className={styles.nodeEditor}
nodeTypes={nodeTypes}
edgeTypes={edgeTypes}
elevateNodesOnSelect
nodesFocusable
attributionPosition={'bottom-left'}
{...editorProps}
>
<Background />
<Controls position={'top-right'} showInteractive={false} fitViewOptions={defaultFitViewOptions} />
</ReactFlow>
);
}
90 changes: 70 additions & 20 deletions frontend/src/components/global-explorer/internal-explorer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { Flex, theme } from 'antd';
import { type MouseEvent, useCallback, useEffect, useState } from 'react';

import { defaultFitViewOptions, NodeEditor } from '@/components/charts/node-editor/node-editor.tsx';
import { CustomEdgeTypes } from '@/components/charts/node-editor/node-types.ts';
import { CustomEdgeTypes, CustomNodeTypes } from '@/components/charts/node-editor/node-types.ts';
import type { Node as GraphNode } from '@/store/api/services/generated/graphApi.ts';
import { NodeType, useGetGraphDataQuery } from '@/store/api/services/generated/graphApi.ts';
import { parseRegularNode } from '@/utils/node-parser.helper';
Expand All @@ -16,9 +16,49 @@ import { parseEdges } from '../explorer/utils';
import { Sidebar, type SidebarFilters } from './sidebar/sidebar';
import { useNodeEditor } from './use-node-editor';

function parseFullNodes(nodes: GraphNode[], setNodeId: (id: string) => void): Node[] {
const DOMAIN_COLORS = [
{ bg: 'rgba(59, 130, 246, 0.08)', border: 'rgba(59, 130, 246, 0.3)' }, // blue
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fine for now, but maybe this should be logged as tech debt as this both hardcodes the colors and will recycle them after 8 domains.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We already use antv from ant design, I believe there might be a colorpallette there. I will look into it and respond here again.

{ bg: 'rgba(16, 185, 129, 0.08)', border: 'rgba(16, 185, 129, 0.3)' }, // green
{ bg: 'rgba(245, 158, 11, 0.08)', border: 'rgba(245, 158, 11, 0.3)' }, // amber
{ bg: 'rgba(139, 92, 246, 0.08)', border: 'rgba(139, 92, 246, 0.3)' }, // purple
{ bg: 'rgba(236, 72, 153, 0.08)', border: 'rgba(236, 72, 153, 0.3)' }, // pink
{ bg: 'rgba(20, 184, 166, 0.08)', border: 'rgba(20, 184, 166, 0.3)' }, // teal
{ bg: 'rgba(249, 115, 22, 0.08)', border: 'rgba(249, 115, 22, 0.3)' }, // orange
{ bg: 'rgba(99, 102, 241, 0.08)', border: 'rgba(99, 102, 241, 0.3)' }, // indigo
];

function parseFullNodes(nodes: GraphNode[], setNodeId: (id: string) => void, domainsEnabled: boolean): Node[] {
// Synthesize domain container nodes
const domainNodes: Node[] = [];
if (domainsEnabled) {
const domainMap = new Map<string, string>();
for (const node of nodes) {
if (node.data.domain_id && node.data.domain && !domainMap.has(node.data.domain_id)) {
domainMap.set(node.data.domain_id, node.data.domain);
}
}
const sortedDomainIds = [...domainMap.keys()].sort();
for (let i = 0; i < sortedDomainIds.length; i++) {
const domainId = sortedDomainIds[i];
const color = DOMAIN_COLORS[i % DOMAIN_COLORS.length];
domainNodes.push({
id: domainId,
position: { x: 0, y: 0 },
type: CustomNodeTypes.DomainNode,
draggable: true,
deletable: false,
data: {
id: domainId,
name: domainMap.get(domainId),
backgroundColor: color.bg,
borderColor: color.border,
},
});
}
}

// Parse regular nodes
return nodes
const regularNodes = nodes
.filter((node) => node.type !== NodeType.DomainNode)
.map((node) => {
let extra_attributes = {};
Expand Down Expand Up @@ -56,9 +96,10 @@ function parseFullNodes(nodes: GraphNode[], setNodeId: (id: string) => void): No
default:
throw new Error(`Unknown node type: ${node.type}`);
}
// For now perma disable domains
return parseRegularNode(node, setNodeId, false, true, extra_attributes);
}); // Skip domain nodes for clarity and reduced clutter
return parseRegularNode(node, setNodeId, domainsEnabled, true, extra_attributes);
});

return [...domainNodes, ...regularNodes];
}

function applyHighlighting(nodes: Node[], edges: Edge[], selectedId: string | null) {
Expand All @@ -83,19 +124,29 @@ function applyHighlighting(nodes: Node[], edges: Edge[], selectedId: string | nu
}
});

// Domain nodes should stay visible if any of their children are connected
const domainNodeIds = new Set(nodes.filter((n) => n.type === CustomNodeTypes.DomainNode).map((n) => n.id));
const activeDomainIds = new Set(
nodes.filter((n) => n.parentId && connectedNodeIds.has(n.id)).map((n) => n.parentId as string),
);

// Apply highlighting to nodes
const highlightedNodes = nodes.map((node) => ({
...node,
data: {
...node.data,
dimmed: !connectedNodeIds.has(node.id),
},
style: {
...node.style,
opacity: connectedNodeIds.has(node.id) ? 1 : 0.3,
zIndex: connectedNodeIds.has(node.id) ? 10 : 1,
},
}));
const highlightedNodes = nodes.map((node) => {
const isConnected =
connectedNodeIds.has(node.id) || (domainNodeIds.has(node.id) && activeDomainIds.has(node.id));
return {
...node,
data: {
...node.data,
dimmed: !isConnected,
},
style: {
...node.style,
opacity: isConnected ? 1 : 0.3,
zIndex: isConnected ? 10 : 1,
},
};
});

// Apply highlighting to edges
const highlightedEdges = edges.map((edge) => ({
Expand Down Expand Up @@ -141,7 +192,7 @@ export default function InternalFullExplorer() {

const generateGraph = useCallback(async () => {
if (graph) {
const nodes = parseFullNodes(graph.nodes, setNodeId);
const nodes = parseFullNodes(graph.nodes, setNodeId, sidebarFilters.domainsEnabled);
const edges = parseEdges(graph.edges, token);

// Explicitly specify straight edge so it doesn't default to default edge (which is a Bézier curve)
Expand Down Expand Up @@ -221,7 +272,6 @@ export default function InternalFullExplorer() {
<Flex className={styles.nodeWrapper}>
<Sidebar
nodes={nodes}
setNodes={setNodes}
onFilterChange={setSidebarFilters}
sidebarFilters={sidebarFilters}
nodeId={nodeId}
Expand Down
11 changes: 9 additions & 2 deletions frontend/src/components/global-explorer/sidebar/sidebar.tsx
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { type Node, useReactFlow } from '@xyflow/react';
import { Flex, Segmented, Select } from 'antd';
import { Flex, Segmented, Select, Switch } from 'antd';
import { type MouseEvent, useCallback, useEffect, useMemo } from 'react';
import { useTranslation } from 'react-i18next';
import type { DataProductContract } from '@/types/data-product';
Expand All @@ -16,7 +16,6 @@ export type SidebarFilters = {

type Props = {
nodes: Node[];
setNodes: (nodes: Node[] | ((nodes: Node[]) => Node[])) => void;
sidebarFilters: SidebarFilters;
onFilterChange: (filters: SidebarFilters) => void;
nodeId: string | null;
Expand Down Expand Up @@ -154,6 +153,14 @@ export function Sidebar({ nodes, sidebarFilters, onFilterChange, nodeId, nodeCli
}
}}
/>
<Flex align="center" gap="small">
<Switch
checked={sidebarFilters.domainsEnabled}
onChange={(checked) => onFilterChange({ ...sidebarFilters, domainsEnabled: checked })}
size="small"
/>
<span>{t('Show Domains')}</span>
</Flex>
<Select
placeholder={t('Select a node')}
value={nodeId}
Expand Down
14 changes: 13 additions & 1 deletion frontend/src/components/global-explorer/use-node-editor.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -68,14 +68,26 @@ const basicLayoutOptions = {
const interDomainLayoutOptions = {
'elk.algorithm': 'layered',
'elk.direction': 'RIGHT',
'elk.spacing.nodeNode': '60.0',
'elk.hierarchyHandling': 'INCLUDE_CHILDREN', // Consider cross-domain edges for global left-to-right flow
'elk.spacing.nodeNode': '80.0',
'elk.layered.spacing.edgeNodeBetweenLayers': '50.0',
'elk.layered.nodePlacement.strategy': 'NETWORK_SIMPLEX',
'elk.layered.crossingMinimization.strategy': 'LAYER_SWEEP',
'elk.portConstraints': 'FIXED_SIDE',
'elk.padding': '[top=50.0,left=50.0,bottom=50.0,right=50.0]',
'elk.aspectRatio': '1.6', // Prefer wider layouts so disconnected domains spread horizontally
'elk.layered.compaction.connectedComponents': 'true', // Compact disconnected components
};

// Layout options within a domain node.
const intraDomainLayoutOptions = {
'elk.algorithm': 'layered',
'elk.direction': 'RIGHT',
'elk.hierarchyHandling': 'INCLUDE_CHILDREN',
'elk.spacing.nodeNode': '40.0',
'elk.layered.spacing.edgeNodeBetweenLayers': '50.0',
'elk.layered.nodePlacement.strategy': 'NETWORK_SIMPLEX',
'elk.portConstraints': 'FIXED_SIDE',

// A bit more padding at the top.
// Most of the time, the label doesn't take up the whole height of the node, so this trick evens it out for short labels (taking up 1 instead of 2 lines).
Expand Down
74 changes: 74 additions & 0 deletions frontend/src/tests/node-parser.helper.test.ts
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

tests should be placed in the tests folder

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No in front end testing, it is preferred to place it next to the component

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

:o that feels weird :D But okay! Learned something new

Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
import { Position } from '@xyflow/react';
import { describe, expect, it, vi } from 'vitest';

import type { Node as GraphNode } from '@/store/api/services/generated/dataProductsApi';
import { parseRegularNode, sharedAttributes } from '@/utils/node-parser.helper';

// Minimal GraphNode factory — only fields the parser actually reads.
// `import type` is erased at compile time so it does not trigger module execution.
const graphNode = (domain_id: string | null = null, domain: string | null = null): GraphNode => ({
id: 'node-1',
type: 'dataProductNode' as GraphNode['type'],
isMain: false,
data: {
id: 'node-1',
name: 'Test Product',
icon_key: 'ANALYTICS',
domain_id,
domain,
description: null,
link_to_id: null,
},
});

const noop = vi.fn();

// ---------------------------------------------------------------------------
// parentId assignment (the core change this PR makes)
// ---------------------------------------------------------------------------

describe('sharedAttributes — domainsEnabled', () => {
it('sets parentId to domain_id when domainsEnabled=true and domain_id is present', () => {
const node = sharedAttributes(graphNode('dom-1', 'Finance'), noop, true, false);
expect(node.parentId).toBe('dom-1');
});

it('does not set parentId when domainsEnabled=false even if domain_id is present', () => {
const node = sharedAttributes(graphNode('dom-1', 'Finance'), noop, false, false);
expect(node.parentId).toBeUndefined();
});

it('does not set parentId when domain_id is null', () => {
const node = sharedAttributes(graphNode(null, null), noop, true, false);
expect(node.parentId).toBeUndefined();
});

it('always sets id, position, type from the input node', () => {
const node = sharedAttributes(graphNode('dom-1', 'Finance'), noop, true, false);
expect(node.id).toBe('node-1');
expect(node.position).toEqual({ x: 0, y: 0 });
expect(node.type).toBe('dataProductNode');
});
});

// ---------------------------------------------------------------------------
// parseRegularNode passes extra_attributes through
// ---------------------------------------------------------------------------

describe('parseRegularNode', () => {
it('merges extra_attributes into data', () => {
const extra = { targetHandlePosition: Position.Left };
const node = parseRegularNode(graphNode('dom-1', 'Finance'), noop, true, false, extra);
expect(node.data.targetHandlePosition).toBe('left');
});

it('preserves parentId set by sharedAttributes', () => {
const node = parseRegularNode(graphNode('dom-1', 'Finance'), noop, true, false, {});
expect(node.parentId).toBe('dom-1');
});

it('exposes the domain name on node.data.domain', () => {
const node = parseRegularNode(graphNode('dom-1', 'Finance'), noop, true, false, {});
expect(node.data.domain).toBe('Finance');
});
});