Skip to content

Commit e434ddb

Browse files
clems4everclems4ever
committed
Populate sources directly from the backend.
This improve performance of the admin UI.
1 parent 62ec335 commit e434ddb

File tree

5 files changed

+133
-102
lines changed

5 files changed

+133
-102
lines changed

‎internal/database/mariadb.go‎

Lines changed: 55 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,13 @@ import (
99
"io"
1010
"reflect"
1111
"strconv"
12+
"strings"
1213
"time"
1314

1415
"github.com/VividCortex/mysqlerr"
1516
"github.com/clems4ever/go-graphkb/internal/knowledge"
1617
"github.com/clems4ever/go-graphkb/internal/schema"
18+
"github.com/clems4ever/go-graphkb/internal/utils"
1719
mysql "github.com/go-sql-driver/mysql"
1820
"github.com/golang-collections/go-datastructures/queue"
1921
"github.com/sirupsen/logrus"
@@ -616,58 +618,85 @@ func (m *MariaDB) Query(ctx context.Context, sql knowledge.SQLTranslation) (*kno
616618
}
617619

618620
func (m *MariaDB) GetAssetSources(ctx context.Context, ids []string) (map[string][]string, error) {
619-
stmt, err := m.db.PrepareContext(ctx, `
620-
SELECT sources.name FROM sources
621-
INNER JOIN assets_by_source ON sources.id = assets_by_source.source_id
622-
WHERE asset_id = ?`)
623-
if err != nil {
624-
return nil, fmt.Errorf("Unable to prepare statement for retrieving asset sources: %w", err)
621+
if len(ids) == 0 {
622+
return nil, nil
625623
}
624+
625+
args := make([]interface{}, len(ids))
626+
for i, id := range ids {
627+
args[i] = id
628+
}
629+
626630
idsSet := make(map[string][]string)
627-
for _, id := range ids {
628-
row, err := stmt.QueryContext(ctx, id)
631+
argsSlices := utils.ChunkSlice(args, 500).([][]interface{})
632+
633+
for _, argsSlice := range argsSlices {
634+
stmt, err := m.db.PrepareContext(ctx, `
635+
SELECT asset_id, sources.name FROM sources
636+
INNER JOIN assets_by_source ON sources.id = assets_by_source.source_id
637+
WHERE asset_id IN (?`+strings.Repeat(",?", len(argsSlice)-1)+`)`)
629638
if err != nil {
630-
return nil, fmt.Errorf("Unable to retrieve sources for asset id %s: %w", id, err)
639+
return nil, fmt.Errorf("Unable to prepare statement for retrieving asset sources: %w", err)
640+
}
641+
row, err := stmt.QueryContext(ctx, argsSlice...)
642+
if err != nil {
643+
return nil, fmt.Errorf("Unable to retrieve sources for assets: %w", err)
631644
}
632-
633-
idsSet[id] = []string{}
634645

635646
var source string
647+
var assetId uint64
648+
636649
for row.Next() {
637-
err = row.Scan(&source)
650+
err = row.Scan(&assetId, &source)
638651
if err != nil {
639652
return nil, fmt.Errorf("Unable to scan row of asset source: %w", err)
640653
}
641-
idsSet[id] = append(idsSet[id], source)
654+
assetIdStr := fmt.Sprintf("%d", assetId)
655+
if _, ok := idsSet[assetIdStr]; !ok {
656+
idsSet[assetIdStr] = []string{}
657+
}
658+
idsSet[assetIdStr] = append(idsSet[assetIdStr], source)
642659
}
643660
}
644661
return idsSet, nil
645662
}
646663

647664
func (m *MariaDB) GetRelationSources(ctx context.Context, ids []string) (map[string][]string, error) {
648-
stmt, err := m.db.PrepareContext(ctx, `
649-
SELECT sources.name FROM sources
650-
INNER JOIN relations_by_source ON sources.id = relations_by_source.source_id
651-
WHERE relation_id = ?`)
652-
if err != nil {
653-
return nil, fmt.Errorf("Unable to prepare statement for retrieving relation sources: %w", err)
665+
if len(ids) == 0 {
666+
return nil, nil
667+
}
668+
args := make([]interface{}, len(ids))
669+
for i, id := range ids {
670+
args[i] = id
654671
}
655672
idsSet := make(map[string][]string)
656-
for _, id := range ids {
657-
row, err := stmt.QueryContext(ctx, id)
673+
674+
argsSlices := utils.ChunkSlice(args, 500).([][]interface{})
675+
676+
for _, argsSlice := range argsSlices {
677+
stmt, err := m.db.PrepareContext(ctx, `
678+
SELECT relation_id, sources.name FROM sources
679+
INNER JOIN relations_by_source ON sources.id = relations_by_source.source_id
680+
WHERE relation_id IN (?`+strings.Repeat(",?", len(argsSlice)-1)+`)`)
658681
if err != nil {
659-
return nil, fmt.Errorf("Unable to retrieve sources for relation id %s: %w", id, err)
682+
return nil, fmt.Errorf("Unable to prepare statement for retrieving relation sources: %w", err)
683+
}
684+
row, err := stmt.QueryContext(ctx, argsSlice...)
685+
if err != nil {
686+
return nil, fmt.Errorf("Unable to retrieve sources for relations: %w", err)
660687
}
661-
662-
idsSet[id] = []string{}
663688

664689
var source string
690+
var relationId string
665691
for row.Next() {
666-
err = row.Scan(&source)
692+
err = row.Scan(&relationId, &source)
667693
if err != nil {
668694
return nil, fmt.Errorf("Unable to scan row of relation source: %w", err)
669695
}
670-
idsSet[id] = append(idsSet[id], source)
696+
if _, ok := idsSet[relationId]; !ok {
697+
idsSet[relationId] = []string{}
698+
}
699+
idsSet[relationId] = append(idsSet[relationId], source)
671700
}
672701
}
673702
return idsSet, nil

‎internal/handlers/handler_query.go‎

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package handlers
33
import (
44
"context"
55
"encoding/json"
6+
"fmt"
67
"net/http"
78
"time"
89

@@ -14,7 +15,8 @@ import (
1415
func PostQuery(database knowledge.GraphDB, queryHistorizer history.Historizer) http.HandlerFunc {
1516
return func(w http.ResponseWriter, r *http.Request) {
1617
type QueryRequestBody struct {
17-
Query string `json:"q"`
18+
Query string `json:"q"`
19+
IncludeSources bool `json:"include_sources"`
1820
}
1921

2022
type ColumnType struct {
@@ -82,6 +84,9 @@ func PostQuery(database knowledge.GraphDB, queryHistorizer history.Historizer) h
8284
})
8385
}
8486

87+
assetIDs := make(map[string]struct{})
88+
relationIDs := make(map[string]struct{})
89+
8590
items := make([][]interface{}, 0)
8691
for res.Cursor.HasMore() {
8792
var d interface{}
@@ -99,15 +104,79 @@ func PostQuery(database knowledge.GraphDB, queryHistorizer history.Historizer) h
99104
switch v := x.(type) {
100105
case knowledge.AssetWithID:
101106
rowDocs = append(rowDocs, v)
107+
if requestBody.IncludeSources {
108+
assetIDs[v.ID] = struct{}{}
109+
}
102110
case knowledge.RelationWithID:
103111
rowDocs = append(rowDocs, v)
112+
if requestBody.IncludeSources {
113+
relationIDs[v.ID] = struct{}{}
114+
}
104115
default:
105116
rowDocs = append(rowDocs, v)
106117
}
107118
}
108119
items = append(items, rowDocs)
109120
}
110121

122+
if requestBody.IncludeSources {
123+
ids := []string{}
124+
for k := range assetIDs {
125+
ids = append(ids, k)
126+
}
127+
sourcesByID, err := database.GetAssetSources(r.Context(), ids)
128+
if err != nil {
129+
ReplyWithInternalError(w, err)
130+
return
131+
}
132+
133+
for i, row := range items {
134+
for j, col := range row {
135+
switch v := col.(type) {
136+
case knowledge.AssetWithID:
137+
sources, ok := sourcesByID[v.ID]
138+
if !ok {
139+
ReplyWithInternalError(w, fmt.Errorf("Unable to find sources of asset with ID %s", v.ID))
140+
return
141+
}
142+
items[i][j] = AssetWithIDAndSources{
143+
AssetWithID: v,
144+
Sources: sources,
145+
}
146+
}
147+
}
148+
}
149+
150+
ids = []string{}
151+
for k := range relationIDs {
152+
ids = append(ids, k)
153+
}
154+
155+
sourcesByID, err = database.GetRelationSources(r.Context(), ids)
156+
if err != nil {
157+
ReplyWithInternalError(w, err)
158+
return
159+
}
160+
161+
for i, row := range items {
162+
for j, col := range row {
163+
switch v := col.(type) {
164+
case knowledge.RelationWithID:
165+
sources, ok := sourcesByID[v.ID]
166+
if !ok {
167+
ReplyWithInternalError(w, fmt.Errorf("Unable to find sources of relation with ID %s", v.ID))
168+
return
169+
}
170+
items[i][j] = RelationWithIDAndSources{
171+
RelationWithID: v,
172+
Sources: sources,
173+
}
174+
}
175+
}
176+
}
177+
178+
}
179+
111180
response := QueryResponseBody{
112181
Items: items,
113182
Columns: columns,

‎internal/handlers/handler_query_sources.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ func postAssetSources(database knowledge.GraphDB, fetcherFn func(context.Context
5656
err = json.NewEncoder(w).Encode(response)
5757
if err != nil {
5858
ReplyWithInternalError(w, err)
59+
return
5960
}
6061
}
6162
}

‎web/src/services/SourceGraph.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import axios from "axios";
22
import { Asset } from "../models/Asset";
3-
import { QueryAssetsSources, QueryRelationsSources, QueryResultSet } from "../models/QueryResultSet";
3+
import { QueryAssetsSources, QueryRelationsSources, QueryResultSet, QueryResultSetWithSources } from "../models/QueryResultSet";
44
import { DatabaseDetails } from "../models/DatabaseDetails";
55

66
export async function getSources() {
@@ -49,8 +49,9 @@ export async function searchAssets(query: string, from: number = 0, size: number
4949
}
5050

5151
export async function postQuery(query: string) {
52-
const res = await axios.post<QueryResultSet>("/api/query", {
52+
const res = await axios.post<QueryResultSetWithSources>("/api/query", {
5353
q: query,
54+
include_sources: true,
5455
}, { validateStatus: s => s === 200 || s === 500 || s === 400 });
5556

5657
if (res.status !== 200) {

‎web/src/views/ExplorerView.tsx‎

Lines changed: 4 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@ import React, { useState, useEffect, useCallback } from 'react';
22
import { makeStyles, Grid, Snackbar, Paper, useTheme } from '@material-ui/core';
33
import GraphExplorer from '../components/GraphExplorer';
44
import QueryField from '../components/QueryField';
5-
import { postQuery, getSources, postAssetsSources, postRelationsSources } from "../services/SourceGraph";
6-
import { QueryAssetsSources, QueryResultSetWithSources } from '../models/QueryResultSet';
5+
import { postQuery, getSources } from "../services/SourceGraph";
6+
import { QueryResultSetWithSources } from '../models/QueryResultSet';
77
import ResultsTable from '../components/ResultsTable';
8-
import { Asset, AssetWithSources } from '../models/Asset';
8+
import { Asset } from '../models/Asset';
99
import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'
1010
import { faProjectDiagram, faDatabase, IconDefinition } from '@fortawesome/free-solid-svg-icons'
1111
import SchemaGraphDialog from '../components/SchemaGraphDialog';
@@ -14,7 +14,6 @@ import DatabaseDialog from '../components/DatabaseDialog';
1414
import SearchField from '../components/SearchField';
1515
import MuiAlert from '@material-ui/lab/Alert';
1616
import { useQueryParam, StringParam, withDefault } from 'use-query-params';
17-
import { Relation, RelationWithSources } from '../models/Relation';
1817

1918
function Alert(props: any) {
2019
return <MuiAlert elevation={6} variant="filled" {...props} />;
@@ -45,75 +44,7 @@ const ExplorerView = () => {
4544
setIsQueryLoading(true);
4645
try {
4746
const res = await postQuery(submittedQuery);
48-
49-
const assetsIds = new Set<string>();
50-
const relationsIds = new Set<string>();
51-
res.items.forEach((row) => {
52-
row.forEach((col, i) => {
53-
if (res.columns[i].type === 'asset') {
54-
const r = col as Asset;
55-
assetsIds.add(r._id);
56-
} else if (res.columns[i].type === 'relation') {
57-
const r = col as Relation;
58-
relationsIds.add(r._id);
59-
}
60-
});
61-
});
62-
63-
64-
// Prepare the queries for all bulks of ids
65-
let remaining = assetsIds.size;
66-
const assetsPromises: Promise<QueryAssetsSources>[] = [];
67-
let ids = Array.from(assetsIds);
68-
while (remaining > 0) {
69-
const first = assetsIds.size - remaining;
70-
const slice = ids.slice(first, first + Math.min(10000, remaining));
71-
assetsPromises.push(postAssetsSources(Array.from(slice)));
72-
remaining -= slice.length;
73-
}
74-
75-
const relationsPromises: Promise<QueryAssetsSources>[] = [];
76-
remaining = relationsIds.size;
77-
ids = Array.from(relationsIds);
78-
while (remaining > 0) {
79-
const first = relationsIds.size - remaining;
80-
const slice = ids.slice(first, first + Math.min(10000, remaining));
81-
relationsPromises.push(postRelationsSources(Array.from(slice)));
82-
remaining -= slice.length;
83-
}
84-
85-
const [resAssets, resRelations] = await Promise.all([Promise.all(assetsPromises), Promise.all(relationsPromises)]);
86-
87-
const assetsSources: {[id: string]: string[]} = {};
88-
const relationsSources: {[id: string]: string[]} = {};
89-
90-
resAssets.forEach(x => Object.keys(x.results).forEach(k => assetsSources[k] = x.results[k]));
91-
resRelations.forEach(x => Object.keys(x.results).forEach(k => relationsSources[k] = x.results[k]));
92-
93-
const items = res.items.map((row) => {
94-
return row.map((it, i) => {
95-
if (res.columns[i].type === 'asset') {
96-
const r = it as Asset;
97-
const sources = assetsSources[r._id];
98-
return {sources: sources, ...r} as AssetWithSources;
99-
}
100-
else if (res.columns[i].type === 'relation') {
101-
const r = it as Relation;
102-
const sources = relationsSources[r._id];
103-
return {sources: sources, ...r} as RelationWithSources;
104-
} else {
105-
return it as string;
106-
}
107-
});
108-
});
109-
110-
const queryResult: QueryResultSetWithSources = {
111-
columns: res.columns,
112-
items: items,
113-
execution_time_ms: res.execution_time_ms,
114-
};
115-
116-
setQueryResult(queryResult);
47+
setQueryResult(res);
11748
} catch (err) {
11849
console.error(err);
11950
setError(err);

0 commit comments

Comments
 (0)