Skip to content
This repository was archived by the owner on Nov 9, 2022. It is now read-only.

Commit f33294d

Browse files
TheLoneRoninalexander-morrisAl
committed
V0.8.2 (#66)
* Optimized tag query and peer route * removing old koi, adding new koi logs :) (#65) * removing old koi, adding new koi logs :) * Delete yarn.lock * fixing order of express operations Co-authored-by: Al <al@openkoi.com> * Formatting * Sub query limit and offset * Update migrations Co-authored-by: Al Morris <37120777+alexander-morris@users.noreply.github.com> Co-authored-by: Al <al@openkoi.com>
1 parent 5408ff5 commit f33294d

File tree

9 files changed

+1587
-609
lines changed

9 files changed

+1587
-609
lines changed

‎bin/index.create.sh‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,22 +13,47 @@ SHOW statement_timeout;
1313
SHOW maintenance_work_mem;
1414
SHOW max_parallel_maintenance_workers;
1515
16+
--- Block Indices
17+
--- Block Height Index
1618
CREATE INDEX "blocks_height" ON blocks USING BTREE ("height");
19+
--- Blocks Mined At Index
1720
CREATE INDEX "blocks_mined_at" ON blocks USING BTREE ("mined_at");
1821
22+
--- Transaction Indices
23+
--- Transaction Height Index
1924
CREATE INDEX "transactions_height" ON transactions USING BTREE ("height");
25+
--- Transaction Target Index
2026
CREATE INDEX "transactions_target" ON transactions USING BTREE ("target");
27+
--- Transaction Owner Address Index
2128
CREATE INDEX "transactions_owner_address" ON transactions USING BTREE ("owner_address");
29+
--- Transaction Namespace Index
2230
CREATE INDEX "index_namespace_transactions" ON transactions USING BTREE ("namespace");
31+
--- Transaction Domain Index
2332
CREATE INDEX "index_domain_transactions" ON transactions USING BTREE ("domain");
33+
--- Transaction App Index
2434
CREATE INDEX "index_app_transactions" ON transactions USING BTREE ("app");
35+
--- Transaction App-Name Index
2536
CREATE INDEX "index_App-Name_transactions" ON transactions USING BTREE ("App-Name");
2637
38+
--- Tag Indices
39+
--- Tag Transaction Id Index
40+
CREATE INDEX "tags_tx_id" ON tags USING BTREE ("tx_id");
41+
--- Tag Name Index (under 64 characters)
2742
CREATE INDEX "tags_name" ON tags USING BTREE ("name") WHERE LENGTH("name") < 64;
43+
--- Tag Value Index (under 64 characters)
2844
CREATE INDEX "tags_value" ON tags USING BTREE ("value") WHERE LENGTH("value") < 64;
45+
--- Tag Name, Value Index (under 64 characters)
2946
CREATE INDEX "tags_name_value" ON tags USING BTREE ("name", "value") WHERE LENGTH("name") < 64 AND LENGTH("value") < 64;
47+
--- Tag Transaction Id, Name Index (under 64 characters)
3048
CREATE INDEX "tags_tx_id_name" ON tags USING BTREE ("tx_id", "name") WHERE LENGTH("name") < 64;
31-
CREATE INDEX "tags_tx_id" ON tags USING BTREE ("tx_id");
49+
--- Tag Name Index (under 128 chracters)
50+
CREATE INDEX "tags_name_128" ON tags USING BTREE ("name") WHERE LENGTH("name") > 64 AND LENGTH("name") < 128;
51+
--- Tag Value Index (under 128 chracters)
52+
CREATE INDEX "tags_value_128" ON tags USING BTREE ("value") WHERE LENGTH("value") > 64; AND LENGTH("value") < 128;
53+
--- Tag Name, Value Index (under 128 chracters)
54+
CREATE INDEX "tags_name_value_128" ON tags USING BTREE ("name", "value") WHERE LENGTH("name") > 64 AND LENGTH("name") < 128 AND LENGTH("value") > 64 AND LENGTH("value") < 128;
55+
--- Tag Transaction Id, Name Index (under 128 chracters)
56+
CREATE INDEX "tags_tx_id_name_128" ON tags USING BTREE ("tx_id", "name") WHERE LENGTH("name") > 64 AND LENGTH("name") < 128;
3257
3358
EOF
3459

‎migrations/20200404025828_initialize.ts‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,6 @@ import {config} from 'dotenv';
44
config();
55

66
export async function up(knex: Knex) {
7-
const indices = JSON.parse(process.env.INDICES || '[]');
8-
97
return knex.schema
108
.withSchema(process.env.ENVIRONMENT || 'public')
119
.createTable('transactions', (table) => {
@@ -26,16 +24,7 @@ export async function up(knex: Knex) {
2624
table.string('parent', 64);
2725
table.timestamp('created_at').defaultTo(knex.fn.now());
2826

29-
for (let i = 0; i < indices.length; i++) {
30-
const index = indices[i];
31-
table.string(index, 64);
32-
table.index(index, `index_${index}_transactions`, 'BTREE');
33-
}
34-
3527
table.primary(['id'], 'pkey_transactions');
36-
table.index(['height'], 'transactions_height', 'BTREE');
37-
table.index(['owner_address'], 'transactions_owner_address', 'BTREE');
38-
table.index(['target'], 'transactions_target', 'BTREE');
3928
})
4029
.createTable('blocks', (table) => {
4130
table.string('id', 64).notNullable();
@@ -47,8 +36,6 @@ export async function up(knex: Knex) {
4736
table.timestamp('created_at').defaultTo(knex.fn.now());
4837

4938
table.primary(['id'], 'pkey_blocks');
50-
table.index(['height'], 'blocks_height', 'BTREE');
51-
table.index(['mined_at'], 'blocks_mined_at', 'BTREE');
5239
})
5340
.createTable('tags', (table) => {
5441
table.string('tx_id', 64).notNullable();
@@ -58,11 +45,6 @@ export async function up(knex: Knex) {
5845
table.timestamp('created_at').defaultTo(knex.fn.now());
5946

6047
table.primary(['tx_id', 'index'], 'pkey_tags');
61-
table.index(['tx_id', 'name'], 'tags_tx_id_name', 'BTREE');
62-
table.index(['name'], 'tags_name', 'BTREE');
63-
table.index(['value'], 'tags_value', 'BTREE');
64-
table.index(['name', 'value'], 'tags_name_value', 'BTREE');
65-
table.index(['tx_id'], 'tags_tx_id', 'BTREE');
6648
});
6749
}
6850

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
"graphql-fields": "^2.0.3",
4646
"js-sha256": "^0.9.0",
4747
"knex": "^0.21.17",
48+
"koi-logs": "git+https://github.com/open-koi/logs.git",
4849
"lodash": "^4.17.20",
4950
"moment": "^2.29.1",
5051
"morgan": "^1.10.0",

‎src/Gateway.ts‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import 'colors';
22
import express, {Express} from 'express';
33
import {config} from 'dotenv';
4-
import cron from 'node-cron';
54
import {corsMiddleware} from './middleware/cors.middleware';
65
import {jsonMiddleware} from './middleware/json.middleware';
76
import {logMiddleware} from './middleware/log.middleware';
@@ -10,8 +9,9 @@ import {graphServer} from './graphql/server.graphql';
109
import {statusRoute} from './route/status.route';
1110
import {proxyRoute} from './route/proxy.route';
1211
import {dataRouteRegex, dataHeadRoute, dataRoute} from './route/data.route';
12+
import {peerRoute} from './route/peer.route';
13+
import {koiLogger, koiLogsRoute, koiLogsRawRoute} from './route/koi.route';
1314
import {startSync} from './database/sync.database';
14-
import {logsHelper, logsTask} from './utility/log.helper';
1515

1616
config();
1717

@@ -22,21 +22,18 @@ export function start() {
2222
app.use(corsMiddleware);
2323
app.use(jsonMiddleware);
2424
app.use(logMiddleware);
25-
26-
cron.schedule('0 0 * * *', async function() {
27-
console.log('running the log cleanup task once per day on ', new Date() );
28-
const result = await logsTask();
29-
console.log('daily log task returned ', result);
30-
});
25+
app.use(koiLogger.logger);
3126

3227
app.get('/', statusRoute);
28+
3329
app.head(dataRouteRegex, dataHeadRoute);
3430
app.get(dataRouteRegex, dataRoute);
3531

3632
graphServer({introspection: true, playground: true}).applyMiddleware({app, path: '/graphql'});
3733

38-
app.get(dataRouteRegex, dataRoute);
39-
app.get('/logs', logsHelper);
34+
app.get('/peers', peerRoute);
35+
app.get('/logs', koiLogsRoute);
36+
app.get('/logs/raw', koiLogsRawRoute);
4037

4138
app.all('*', proxyRoute);
4239

‎src/graphql/query.graphql.ts‎

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import {indices} from '../utility/order.utility';
44
import {connection} from '../database/connection.database';
55
import {ISO8601DateTimeString} from '../utility/encoding.utility';
66
import {TagFilter} from './types';
7-
import {tagToB64} from '../query/transaction.query';
7+
import {tagToB64, toB64url} from '../query/transaction.query';
88

99
config();
1010

@@ -72,13 +72,19 @@ export async function generateQuery(params: QueryParams): Promise<QueryBuilder>
7272
if (tags) {
7373
const tagsConverted = tagToB64(tags);
7474

75+
const subQuery = connection
76+
.queryBuilder()
77+
.select('*')
78+
.from('tags');
79+
80+
let runSubQuery = false;
81+
7582
for (let i = 0; i < tagsConverted.length; i++) {
7683
const tag = tagsConverted[i];
77-
const tagAlias = `${i}_${i}`;
7884
let indexed = false;
7985

8086
for (let ii = 0; ii < indices.length; ii++) {
81-
const index = indices[ii];
87+
const index = toB64url(indices[ii]);
8288

8389
if (tag.name === index) {
8490
indexed = true;
@@ -87,13 +93,22 @@ export async function generateQuery(params: QueryParams): Promise<QueryBuilder>
8793
}
8894

8995
if (indexed === false) {
90-
query.join(`tags as ${tagAlias}`, (join) => {
91-
join.on('transactions.id', `${tagAlias}.tx_id`);
96+
subQuery.where('name', tag.name);
97+
subQuery.whereIn('value', tag.values);
98+
runSubQuery = true;
99+
}
100+
}
101+
102+
if (runSubQuery) {
103+
const results = await subQuery.limit(limit).offset(offset);
104+
const tx_ids = [];
92105

93-
join.andOnIn(`${tagAlias}.name`, [tag.name]);
94-
join.andOnIn(`${tagAlias}.value`, tag.values);
95-
});
106+
for (let i = 0; i < results.length; i++) {
107+
const {tx_id} = results[i];
108+
tx_ids.push(tx_id);
96109
}
110+
111+
query.whereIn('transactions.id', tx_ids);
97112
}
98113
}
99114

‎src/route/koi.route.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
import KoiLogs from 'koi-logs';
2+
import {Request, Response} from 'express';
3+
4+
export const koiLogger = new KoiLogs('./');
5+
6+
export async function koiLogsRoute(req: Request, res: Response) {
7+
return koiLogger.koiLogsHelper(req, res);
8+
}
9+
10+
export async function koiLogsRawRoute(req: Request, res: Response) {
11+
return koiLogger.koiRawLogsHelper(req, res);
12+
}

‎src/route/peer.route.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import {Request, Response} from 'express';
2+
3+
export async function peerRoute(req: Request, res: Response) {
4+
return res.status(200).send({
5+
status: 'OK',
6+
nodes: [
7+
'https://gateway-n1.amplify.host',
8+
'https://gateway-n2.amplify.host',
9+
],
10+
});
11+
}

‎src/utility/log.helper.ts‎

Lines changed: 0 additions & 151 deletions
This file was deleted.

0 commit comments

Comments
 (0)