Skip to content

Commit bf23cd6

Browse files
BraydenBrayden
authored
Merge pull request #37 from Brayden/bwilmoth/internal-external-sources
Support both internal and external data sources
2 parents af01346 + ca65858 commit bf23cd6

File tree

15 files changed

+595
-557
lines changed

15 files changed

+595
-557
lines changed

‎README.md‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
<li><strong><a href="https://github.com/Brayden/starbasedb/edit/main/README.md#deploy-a-starbasedb">Database Interface</a></strong> included out of the box deployed with your Cloudflare Worker</li>
2929
<li><strong><a href="https://starbasedb.hashnode.space/default-guide/import-export/sql-dump">Import & Export Data</a></strong> to import & extract your schema and data into a local `.sql`, `.json` or `.csv` file</li>
3030
<li><strong><a href="https://github.com/Brayden/starbasedb/pull/26">Bindable Microservices</a></strong> via templates to kickstart development and own the logic (e.g. user authentication)</li>
31+
<li><strong><a href="https://starbasedb.hashnode.space/default-guide/introduction/connect-external-database">Connect to External Databases</a></strong> such as Postgres and MySQL and access it with the methods above</li>
3132
<li><strong>Scale-to-zero Compute</strong> to reduce costs when your database is not in use</li>
3233
</ul>
3334
<br />
@@ -224,36 +225,33 @@ You can request a `database_dump.sql` file that exports your database schema and
224225
<pre>
225226
<code>
226227
curl --location 'https://starbasedb.YOUR-ID-HERE.workers.dev/export/dump' \
227-
--header 'Authorization: Bearer ABC123'
228+
--header 'Authorization: Bearer ABC123' \
228229
--output database_dump.sql
229230
</code>
230231
</pre>
231232

232233
<h3>JSON Data Export</h3>
233234
<pre>
234235
<code>
235-
curl
236-
--location 'https://starbasedb.YOUR-ID-HERE.workers.dev/export/json/users' \
237-
--header 'Authorization: Bearer ABC123'
236+
curl --location 'https://starbasedb.YOUR-ID-HERE.workers.dev/export/json/users' \
237+
--header 'Authorization: Bearer ABC123' \
238238
--output output.json
239239
</code>
240240
</pre>
241241

242242
<h3>CSV Data Export</h3>
243243
<pre>
244244
<code>
245-
curl
246-
--location 'https://starbasedb.YOUR-ID-HERE.workers.dev/export/csv/users' \
247-
--header 'Authorization: Bearer ABC123'
245+
curl --location 'https://starbasedb.YOUR-ID-HERE.workers.dev/export/csv/users' \
246+
--header 'Authorization: Bearer ABC123' \
248247
--output output.csv
249248
</code>
250249
</pre>
251250

252251
<h3>SQL Import</h3>
253252
<pre>
254253
<code>
255-
curl
256-
--location 'https://starbasedb.YOUR-ID-HERE.workers.dev/import/dump' \
254+
curl --location 'https://starbasedb.YOUR-ID-HERE.workers.dev/import/dump' \
257255
--header 'Authorization: Bearer ABC123' \
258256
--form 'sqlFile=@"./Desktop/sqldump.sql"'
259257
</code>

‎src/do.ts‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
import { DurableObject } from "cloudflare:workers";
2+
import { OperationQueueItem, QueryResponse } from "./operation";
3+
import { createResponse } from "./utils";
4+
5+
export class DatabaseDurableObject extends DurableObject {
6+
// Durable storage for the SQL database
7+
public sql: SqlStorage;
8+
public storage: DurableObjectStorage;
9+
10+
// Queue of operations to be processed, with each operation containing a list of queries to be executed
11+
private operationQueue: Array<OperationQueueItem> = [];
12+
13+
// Flag to indicate if an operation is currently being processed
14+
private processingOperation: { value: boolean } = { value: false };
15+
16+
/**
17+
* The constructor is invoked once upon creation of the Durable Object, i.e. the first call to
18+
* `DurableObjectStub::get` for a given identifier (no-op constructors can be omitted)
19+
*
20+
* @param ctx - The interface for interacting with Durable Object state
21+
* @param env - The interface to reference bindings declared in wrangler.toml
22+
*/
23+
constructor(ctx: DurableObjectState, env: Env) {
24+
super(ctx, env);
25+
this.sql = ctx.storage.sql;
26+
this.storage = ctx.storage;
27+
}
28+
29+
/**
30+
* Execute a raw SQL query on the database, typically used for external requests
31+
* from other service bindings (e.g. auth). This serves as an exposed function for
32+
* other service bindings to query the database without having to have knowledge of
33+
* the current operation queue or processing state.
34+
*
35+
* @param sql - The SQL query to execute.
36+
* @param params - Optional parameters for the SQL query.
37+
* @returns A response containing the query result or an error message.
38+
*/
39+
async executeExternalQuery(sql: string, params: any[] | undefined): Promise<any> {
40+
try {
41+
const queries = [{ sql, params }];
42+
const response = await this.enqueueOperation(
43+
queries,
44+
false,
45+
false,
46+
this.operationQueue,
47+
() => this.processNextOperation(this.sql, this.operationQueue, this.ctx, this.processingOperation)
48+
);
49+
50+
return response;
51+
} catch (error: any) {
52+
console.error('Execute External Query Error:', error);
53+
return null;
54+
}
55+
}
56+
57+
public executeQuery(sql: string, params: any[] | undefined, isRaw: boolean): QueryResponse {
58+
try {
59+
let cursor;
60+
61+
if (params && params.length) {
62+
cursor = this.sql.exec(sql, ...params);
63+
} else {
64+
cursor = this.sql.exec(sql);
65+
}
66+
67+
let result;
68+
69+
if (isRaw) {
70+
result = {
71+
columns: cursor.columnNames,
72+
rows: Array.from(cursor.raw()),
73+
meta: {
74+
rows_read: cursor.rowsRead,
75+
rows_written: cursor.rowsWritten,
76+
},
77+
};
78+
} else {
79+
result = cursor.toArray();
80+
}
81+
82+
return result;
83+
} catch (error) {
84+
console.error('SQL Execution Error:', error);
85+
throw error;
86+
}
87+
}
88+
89+
public executeTransaction(queries: { sql: string; params?: any[] }[], isRaw: boolean): any[] {
90+
return this.storage.transactionSync(() => {
91+
const results = [];
92+
93+
try {
94+
for (const queryObj of queries) {
95+
const { sql, params } = queryObj;
96+
const result = this.executeQuery(sql, params, isRaw);
97+
results.push(result);
98+
}
99+
100+
return results;
101+
} catch (error) {
102+
console.error('Transaction Execution Error:', error);
103+
throw error;
104+
}
105+
});
106+
}
107+
108+
enqueueOperation(
109+
queries: { sql: string; params?: any[] }[],
110+
isTransaction: boolean,
111+
isRaw: boolean,
112+
operationQueue: any[],
113+
processNextOperation: () => Promise<void>
114+
): Promise<{ result?: any, error?: string | undefined, status: number }> {
115+
const MAX_WAIT_TIME = 25000;
116+
return new Promise((resolve, reject) => {
117+
const timeout = setTimeout(() => {
118+
reject(createResponse(undefined, 'Operation timed out.', 503));
119+
}, MAX_WAIT_TIME);
120+
121+
operationQueue.push({
122+
queries,
123+
isTransaction,
124+
isRaw,
125+
resolve: (value: any) => {
126+
clearTimeout(timeout);
127+
128+
resolve({
129+
result: value,
130+
error: undefined,
131+
status: 200
132+
})
133+
},
134+
reject: (reason?: any) => {
135+
clearTimeout(timeout);
136+
137+
reject({
138+
result: undefined,
139+
error: reason ?? 'Operation failed.',
140+
status: 500
141+
})
142+
}
143+
});
144+
145+
processNextOperation().catch((err) => {
146+
console.error('Error processing operation queue:', err);
147+
});
148+
});
149+
}
150+
151+
async processNextOperation(
152+
sqlInstance: any,
153+
operationQueue: OperationQueueItem[],
154+
ctx: any,
155+
processingOperation: { value: boolean }
156+
) {
157+
if (processingOperation.value) {
158+
// Already processing an operation
159+
return;
160+
}
161+
162+
if (operationQueue.length === 0) {
163+
// No operations remaining to process
164+
return;
165+
}
166+
167+
processingOperation.value = true;
168+
const { queries, isTransaction, isRaw, resolve, reject } = operationQueue.shift()!;
169+
170+
try {
171+
let result;
172+
173+
if (isTransaction) {
174+
result = await this.executeTransaction(queries, isRaw);
175+
} else {
176+
const { sql, params } = queries[0];
177+
result = this.executeQuery(sql, params, isRaw);
178+
}
179+
180+
resolve(result);
181+
} catch (error: any) {
182+
reject(error.message || 'Operation failed.');
183+
} finally {
184+
processingOperation.value = false;
185+
await this.processNextOperation(sqlInstance, operationQueue, ctx, processingOperation);
186+
}
187+
}
188+
}

‎src/export/csv.ts‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,13 @@
11
import { getTableData, createExportResponse } from './index';
22
import { createResponse } from '../utils';
3+
import { DataSource } from '..';
34

45
export async function exportTableToCsvRoute(
5-
sql: any,
6-
operationQueue: any,
7-
ctx: any,
8-
processingOperation: { value: boolean },
9-
tableName: string
6+
tableName: string,
7+
dataSource: DataSource
108
): Promise<Response> {
119
try {
12-
const data = await getTableData(sql, operationQueue, ctx, processingOperation, tableName);
10+
const data = await getTableData(tableName, dataSource);
1311

1412
if (data === null) {
1513
return createResponse(undefined, `Table '${tableName}' does not exist.`, 404);

‎src/export/dump.ts‎

Lines changed: 10 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,31 @@
1-
import { enqueueOperation, processNextOperation } from '../operation';
1+
import { executeOperation } from '.';
2+
import { DataSource } from '..';
23
import { createResponse } from '../utils';
34

45
export async function dumpDatabaseRoute(
5-
sql: any,
6-
operationQueue: any,
7-
ctx: any,
8-
processingOperation: { value: boolean }
6+
dataSource: DataSource
97
): Promise<Response> {
108
try {
119
// Get all table names
12-
const tablesResult = await enqueueOperation(
13-
[{ sql: "SELECT name FROM sqlite_master WHERE type='table';" }],
14-
false,
15-
false,
16-
operationQueue,
17-
() => processNextOperation(sql, operationQueue, ctx, processingOperation)
18-
);
10+
const tablesResult = await executeOperation([{ sql: "SELECT name FROM sqlite_master WHERE type='table';" }], dataSource)
1911

20-
const tables = tablesResult.result.map((row: any) => row.name);
12+
const tables = tablesResult.map((row: any) => row.name);
2113
let dumpContent = "SQLite format 3\0"; // SQLite file header
2214

2315
// Iterate through all tables
2416
for (const table of tables) {
2517
// Get table schema
26-
const schemaResult = await enqueueOperation(
27-
[{ sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';` }],
28-
false,
29-
false,
30-
operationQueue,
31-
() => processNextOperation(sql, operationQueue, ctx, processingOperation)
32-
);
18+
const schemaResult = await executeOperation([{ sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';` }], dataSource)
3319

34-
if (schemaResult.result.length) {
35-
const schema = schemaResult.result[0].sql;
20+
if (schemaResult.length) {
21+
const schema = schemaResult[0].sql;
3622
dumpContent += `\n-- Table: ${table}\n${schema};\n\n`;
3723
}
3824

3925
// Get table data
40-
const dataResult = await enqueueOperation(
41-
[{ sql: `SELECT * FROM ${table};` }],
42-
false,
43-
false,
44-
operationQueue,
45-
() => processNextOperation(sql, operationQueue, ctx, processingOperation)
46-
);
26+
const dataResult = await executeOperation([{ sql: `SELECT * FROM ${table};` }], dataSource)
4727

48-
for (const row of dataResult.result) {
28+
for (const row of dataResult) {
4929
const values = Object.values(row).map(value =>
5030
typeof value === 'string' ? `'${value.replace(/'/g, "''")}'` : value
5131
);

‎src/export/index.ts‎

Lines changed: 14 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,26 @@
1-
import { enqueueOperation, processNextOperation } from '../operation';
1+
import { DataSource } from "..";
2+
import { executeTransaction } from "../operation";
3+
4+
export async function executeOperation(queries: { sql: string, params?: any[] }[], dataSource: DataSource): Promise<any> {
5+
const results: any[] = (await executeTransaction(queries, false, dataSource)) as any[];
6+
return results?.length > 0 ? results[0] : undefined;
7+
}
28

39
export async function getTableData(
4-
sql: any,
5-
operationQueue: any,
6-
ctx: any,
7-
processingOperation: { value: boolean },
8-
tableName: string
10+
tableName: string,
11+
dataSource: DataSource
912
): Promise<any[] | null> {
1013
try {
1114
// Verify if the table exists
12-
const tableExistsResult = await enqueueOperation(
13-
[{ sql: `SELECT name FROM sqlite_master WHERE type='table' AND name=?;`, params: [tableName] }],
14-
false,
15-
false,
16-
operationQueue,
17-
() => processNextOperation(sql, operationQueue, ctx, processingOperation)
18-
);
19-
20-
if (tableExistsResult.result.length === 0) {
15+
const tableExistsResult = await executeOperation([{ sql: `SELECT name FROM sqlite_master WHERE type='table' AND name=?;`, params: [tableName] }], dataSource)
16+
17+
if (tableExistsResult.length === 0) {
2118
return null;
2219
}
2320

2421
// Get table data
25-
const dataResult = await enqueueOperation(
26-
[{ sql: `SELECT * FROM ${tableName};` }],
27-
false,
28-
false,
29-
operationQueue,
30-
() => processNextOperation(sql, operationQueue, ctx, processingOperation)
31-
);
32-
33-
return dataResult.result;
22+
const dataResult = await executeOperation([{ sql: `SELECT * FROM ${tableName};` }], dataSource)
23+
return dataResult;
3424
} catch (error: any) {
3525
console.error('Table Data Fetch Error:', error);
3626
throw error;

‎src/export/json.ts‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,13 @@
11
import { getTableData, createExportResponse } from './index';
22
import { createResponse } from '../utils';
3+
import { DataSource } from '..';
34

45
export async function exportTableToJsonRoute(
5-
sql: any,
6-
operationQueue: any,
7-
ctx: any,
8-
processingOperation: { value: boolean },
9-
tableName: string
6+
tableName: string,
7+
dataSource: DataSource
108
): Promise<Response> {
119
try {
12-
const data = await getTableData(sql, operationQueue, ctx, processingOperation, tableName);
10+
const data = await getTableData(tableName, dataSource);
1311

1412
if (data === null) {
1513
return createResponse(undefined, `Table '${tableName}' does not exist.`, 404);

0 commit comments

Comments
 (0)