11import fetch from "node-fetch" ;
22
3- export async function createBackup (
4- options : { accountId : string ; databaseId : string ; apiKey : string } ,
5- ) {
3+ export async function createBackup ( options : {
4+ accountId : string ;
5+ databaseId : string ;
6+ apiKey : string ;
7+ // Default to 1000
8+ limit ?: number ;
9+ } ) {
10+ const limit = options . limit ?? 1000 ;
611 const lines : string [ ] = [ ] ;
712 function append ( command : string ) {
813 lines . push ( command ) ;
914 }
1015
11- async function fetchD1 ( sql : string , params ?: unknown [ ] ) {
16+ async function fetchD1 < T = Record < string , unknown > > (
17+ sql : string ,
18+ params ?: unknown [ ] ,
19+ ) {
1220 const response = await fetch (
1321 `https://api.cloudflare.com/client/v4/accounts/${ options . accountId } ${ options . databaseId } ,
1422 {
@@ -29,22 +37,21 @@ export async function createBackup(
2937 throw new Error (
3038 `D1 Error: ${ body . errors
3139 . map ( ( error : { message : string } ) => error . message )
32- . join ( ", " )
33- } `,
40+ . join ( ", " ) } `,
3441 ) ;
3542 }
3643
3744 return body . result as {
3845 meta : { } ;
39- results : Record < string , unknown > [ ] ;
46+ results : T [ ] ;
4047 success : boolean ;
4148 } [ ] ;
4249 }
4350
4451 let writableSchema : boolean = false ;
4552
4653 {
47- const [ tables ] = await fetchD1 (
54+ const [ tables ] = await fetchD1 < { name : string ; type : string ; sql : string } > (
4855 "SELECT name, type, sql FROM sqlite_master WHERE sql IS NOT NULL AND type = 'table' ORDER BY name" ,
4956 ) ;
5057
@@ -53,7 +60,6 @@ export async function createBackup(
5360 console . warn ( `Table name is not string: ${ table . name } ) ;
5461 continue ;
5562 }
56-
5763 if ( table . name . startsWith ( "_cf_" ) ) {
5864 continue ; // we're not allowed access to these
5965 } else if ( table . name === "sqlite_sequence" ) {
@@ -75,8 +81,10 @@ export async function createBackup(
7581 const tableName = table . name . replace ( "'" , "''" ) ;
7682
7783 append (
78- `INSERT INTO sqlite_master (type, name, tbl_name, rootpage, sql) VALUES ('table', '${ tableName } ${ tableName } ${ table . sql . replace ( / ' / g, "''" )
79- } ');`,
84+ `INSERT INTO sqlite_master (type, name, tbl_name, rootpage, sql) VALUES ('table', '${ tableName } ${ tableName } ${ table . sql . replace (
85+ / ' / g,
86+ "''" ,
87+ ) } ');`,
8088 ) ;
8189
8290 continue ;
@@ -85,8 +93,9 @@ export async function createBackup(
8593 table . sql . toUpperCase ( ) . startsWith ( "CREATE TABLE " )
8694 ) {
8795 append (
88- `CREATE TABLE IF NOT EXISTS ${ table . sql . substring ( "CREATE TABLE " . length )
89- } ;`,
96+ `CREATE TABLE IF NOT EXISTS ${ table . sql . substring (
97+ "CREATE TABLE " . length ,
98+ ) } ;`,
9099 ) ;
91100 } else {
92101 append ( `${ table . sql } ) ;
@@ -106,47 +115,72 @@ export async function createBackup(
106115 if ( tableRow . results [ 0 ] ) {
107116 const columnNames = Object . keys ( tableRow . results [ 0 ] ) ;
108117
109- const queries = [ ] ;
110-
111- // D1 said maximum depth is 20, but the limit is seemingly at 9.
112- for ( let index = 0 ; index < columnNames . length ; index += 9 ) {
113- const currentColumnNames = columnNames . slice (
114- index ,
115- Math . min ( index + 9 , columnNames . length ) ,
116- ) ;
118+ const [ tableRowCount ] = await fetchD1 < { count : number } > (
119+ `SELECT COUNT(*) AS count FROM "${ tableNameIndent } ,
120+ ) ;
117121
118- queries . push (
119- `SELECT '${ currentColumnNames . map ( ( columnName ) =>
120- `'||quote("${ columnName . replace ( '"' , '""' ) }
121- ) . join ( ", " )
122- } ' AS partialCommand FROM "${ tableNameIndent } ,
123- ) ;
122+ if ( tableRowCount === null ) {
123+ throw new Error ( "Failed to get table row count from table." ) ;
124124 }
125125
126- const results = await fetchD1 ( queries . join ( ";" ) ) ;
126+ for (
127+ let offset = 0 ;
128+ offset <= tableRowCount . results [ 0 ] . count ;
129+ offset += limit
130+ ) {
131+ const queries = [ ] ;
132+
133+ // D1 said maximum depth is 20, but the limit is seemingly at 9.
134+ for ( let index = 0 ; index < columnNames . length ; index += 9 ) {
135+ const currentColumnNames = columnNames . slice (
136+ index ,
137+ Math . min ( index + 9 , columnNames . length ) ,
138+ ) ;
127139
128- if ( results . length && results [ 0 ] . results . length ) {
129- for ( let result = 1 ; result < results . length ; result ++ ) {
130- if (
131- results [ result ] . results . length !== results [ 0 ] . results . length
132- ) {
133- throw new Error (
134- "Failed to split expression tree into several queries properly." ,
135- ) ;
136- }
140+ queries . push (
141+ `SELECT '${ currentColumnNames
142+ . map (
143+ ( columnName ) =>
144+ `'||quote("${ columnName . replace ( '"' , '""' ) } ,
145+ )
146+ . join (
147+ ", " ,
148+ ) } ' AS partialCommand FROM "${ tableNameIndent } ${ limit } ${ offset } ,
149+ ) ;
137150 }
138151
139- for ( let row = 0 ; row < results [ 0 ] . results . length ; row ++ ) {
140- let columns = [ ] ;
152+ const results = await fetchD1 < { partialCommand : string } > (
153+ queries . join ( ";\n" ) ,
154+ ) ;
141155
142- for ( let result = 0 ; result < results . length ; result ++ ) {
143- columns . push ( results [ result ] . results [ row ] . partialCommand ) ;
156+ if ( results . length && results [ 0 ] . results . length ) {
157+ for ( let result = 1 ; result < results . length ; result ++ ) {
158+ if (
159+ results [ result ] . results . length !== results [ 0 ] . results . length
160+ ) {
161+ throw new Error (
162+ "Failed to split expression tree into several queries properly." ,
163+ ) ;
164+ }
144165 }
145166
146- append (
147- `INSERT INTO "${ tableNameIndent } ${ columnNames . map ( ( columnName ) => `"${ columnName } ) . join ( ", " )
148- } ) VALUES (${ columns . join ( ", " ) } ,
149- ) ;
167+ for ( let row = 0 ; row < results [ 0 ] . results . length ; row ++ ) {
168+ let columns : string [ ] = [ ] ;
169+
170+ for ( let result = 0 ; result < results . length ; result ++ ) {
171+ columns . push (
172+ results [ result ] . results [ row ] . partialCommand as string ,
173+ ) ;
174+ }
175+
176+ append (
177+ `INSERT INTO "${ tableNameIndent } ${ columnNames
178+ . map ( ( columnName ) => `"${ columnName } )
179+ . join ( ", " ) } ) VALUES (${ columns
180+ . map ( ( column ) => column . replace ( "\n" , "\\n" ) )
181+ . join ( ", " ) } );`,
182+ ) ;
183+ }
150184 }
151185 }
152186 }
0 commit comments