Skip to content

Commit 2ae2da7

Browse files
committed
Fix large SQL execution for direct Cloudflare D1.
1 parent 42958e7 commit 2ae2da7

10 files changed

Lines changed: 468 additions & 19 deletions

File tree

‎docs/reference/cli/database.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ By default, targets the local database. Pass `--remote` to run against the deplo
6464
- **Hosted PostgreSQL and MySQL projects**: fetches the stored connection string from the platform and connects directly.
6565
- **Direct Cloudflare PostgreSQL and MySQL projects**: uses `DATABASE_URL` from the current shell; Cloudflare cannot return the password from Hyperdrive.
6666

67+
On direct Cloudflare D1 deployments, `--file` uses bulk import. Void waits for the import to finish and reports failures; bulk imports do not return individual query rows. Pass SQL inline to return query rows.
68+
6769
For destructive statements (`DELETE`, `UPDATE`, `DROP`, etc.) when running in a TTY, you will be prompted to confirm before the query is sent to the deployed database. Non-TTY environments (CI) skip the prompt.
6870

6971
## `void db migrate` {#void-db-migrate}

‎packages/deploy-cloudflare/src/operations.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,11 @@ export type CloudflareOperation =
1111
} & ConfigTarget)
1212
| ({ type: 'd1.info'; name: string } & ConfigTarget)
1313
| { type: 'd1.create'; name: string; binding: string; configPath: string }
14-
| { type: 'd1.execute'; database: string; sql: string; configPath: string }
14+
| ({ type: 'd1.execute'; database: string; configPath: string } & (
15+
| { sql: string; file?: never }
16+
| { file: string; sql?: never }
17+
))
18+
| { type: 'd1.import'; database: string; file: string; configPath: string }
1519
| { type: 'd1.migrations.list' | 'd1.migrations.apply'; database: string; configPath: string }
1620
| { type: 'd1.time-travel.bookmark'; database: string; configPath: string }
1721
| {
Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
import { createHash } from 'node:crypto';
2+
import { openAsBlob } from 'node:fs';
3+
import { setTimeout } from 'node:timers/promises';
4+
import { sanitizeTerminalLogText } from './terminal-log-text.ts';
5+
6+
type Json = Record<string, unknown>;
7+
8+
function object(value: unknown): Json {
9+
return value && typeof value === 'object' && !Array.isArray(value) ? (value as Json) : {};
10+
}
11+
12+
/** Upload SQL separately from process arguments and wait for D1 to finish ingesting it. */
13+
export async function importD1SqlFile({
14+
endpoint,
15+
headers,
16+
file,
17+
}: {
18+
endpoint: string;
19+
headers: Record<string, string>;
20+
file: string;
21+
}): Promise<void> {
22+
const signal = AbortSignal.timeout(10 * 60_000);
23+
const blob = await openAsBlob(file);
24+
const hash = createHash('md5');
25+
for await (const chunk of blob.stream()) {
26+
hash.update(chunk);
27+
}
28+
const etag = hash.digest('hex');
29+
30+
async function request(body: Json): Promise<Json> {
31+
const response = await fetch(endpoint, {
32+
method: 'POST',
33+
headers: { ...headers, 'Content-Type': 'application/json' },
34+
body: JSON.stringify(body),
35+
redirect: 'error',
36+
signal,
37+
});
38+
const data = object(await response.json());
39+
if (!response.ok || data.success !== true) {
40+
const errors = Array.isArray(data.errors) ? data.errors : [];
41+
const detail = errors
42+
.map((error) => String(object(error).message ?? ''))
43+
.filter(Boolean)
44+
.join('; ');
45+
throw new Error(
46+
`D1 import failed (HTTP ${response.status})${detail ? ': ' + sanitizeTerminalLogText(detail) : '.'}`,
47+
);
48+
}
49+
const state = object(data.result);
50+
if (state.status === 'error' || state.success === false) {
51+
throw new Error(
52+
`D1 import failed: ${sanitizeTerminalLogText(String(state.error ?? 'Unknown import error.'))}`,
53+
);
54+
}
55+
return state;
56+
}
57+
58+
const initialized = await request({ action: 'init', etag });
59+
let state = initialized;
60+
if (!initialized.at_bookmark) {
61+
if (
62+
typeof initialized.filename !== 'string' ||
63+
!initialized.filename ||
64+
typeof initialized.upload_url !== 'string'
65+
) {
66+
throw new Error('Cloudflare returned an invalid D1 import upload.');
67+
}
68+
let uploadUrl: URL;
69+
try {
70+
uploadUrl = new URL(initialized.upload_url);
71+
} catch {
72+
throw new Error('Cloudflare returned an invalid D1 import upload URL.');
73+
}
74+
const localApi = new URL(endpoint);
75+
if (
76+
uploadUrl.protocol !== 'https:' &&
77+
!(
78+
uploadUrl.protocol === 'http:' &&
79+
localApi.protocol === 'http:' &&
80+
['127.0.0.1', '[::1]', 'localhost'].includes(uploadUrl.hostname) &&
81+
uploadUrl.hostname === localApi.hostname
82+
)
83+
) {
84+
throw new Error('Cloudflare returned an insecure D1 import upload URL.');
85+
}
86+
// The presigned URL authorizes this request; never forward the account's credentials.
87+
const upload = await fetch(uploadUrl, {
88+
method: 'PUT',
89+
body: blob,
90+
redirect: 'error',
91+
signal,
92+
});
93+
await upload.body?.cancel();
94+
if (!upload.ok || upload.headers.get('etag')?.replaceAll('"', '') !== etag) {
95+
throw new Error(`D1 import upload failed verification (HTTP ${upload.status}).`);
96+
}
97+
state = await request({ action: 'ingest', etag, filename: initialized.filename });
98+
}
99+
100+
for (;;) {
101+
if (state.status === 'complete' && state.success === true) {
102+
return;
103+
}
104+
if (typeof state.at_bookmark !== 'string' || !state.at_bookmark) {
105+
throw new Error('Cloudflare returned an invalid D1 import status.');
106+
}
107+
await setTimeout(1000, undefined, { signal });
108+
state = await request({ action: 'poll', current_bookmark: state.at_bookmark });
109+
}
110+
}

‎packages/void/src/cli/cloudflare-operation-process.ts‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import {
2020
readCfUploadOutput,
2121
safePositional,
2222
} from './cf-command-values.ts';
23+
import { importD1SqlFile } from './cf-d1-import.ts';
2324
import { persistCfKeyringPreference, readCfSessionCredential } from './cf-session.ts';
2425
import { getCfCliVersion, getVoidUserAgent } from './cf-user-agent.ts';
2526
import { publishCloudflareWorkerSettings } from './cf-worker-settings.ts';
@@ -717,6 +718,15 @@ async function readAllQueues(): Promise<Array<Json>> {
717718

718719
async function resources(): Promise<Result> {
719720
switch (operation.type) {
721+
case 'd1.import': {
722+
const database = d1Database(operation.database);
723+
await importD1SqlFile({
724+
endpoint: `${apiBaseUrl()}/accounts/${encodeURIComponent(accountId())}/d1/database/${encodeURIComponent(database.id)}/import`,
725+
headers: await apiHeaders(),
726+
file: operation.file,
727+
});
728+
return result(0, '', '', []);
729+
}
720730
case 'workers.subdomain.get': {
721731
const body = await cloudflareApiGet(
722732
`/accounts/${encodeURIComponent(accountId())}/workers/subdomain`,
@@ -730,8 +740,11 @@ async function resources(): Promise<Result> {
730740
}
731741
case 'd1.execute': {
732742
const database = d1Database(operation.database);
733-
// One argv item, so cf cannot read SQL that starts with "-" (such as a comment) as a flag.
734-
return runCf(['d1', 'query', database.id, `--sql=${operation.sql}`]);
743+
const sql =
744+
operation.file !== undefined ? readFileSync(operation.file, 'utf8') : operation.sql;
745+
return withSecretFile(JSON.stringify({ sql }), (path) =>
746+
runCf(['d1', 'query', database.id, '--body', '@' + path]),
747+
);
735748
}
736749
case 'd1.migrations.list':
737750
case 'd1.migrations.apply': {

‎packages/void/src/cli/cloudflare-operations.ts‎

Lines changed: 39 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import { join } from 'pathslash';
1717
import { materializeCloudflareConfig, persistCloudflareConfig } from '../cloudflare-config.ts';
1818
import { createCloudflareProcessRunner } from './cloudflare-process.ts';
1919
import { findAmbientConfigOverride, findWranglerCiOverrideDotenvFile } from './deploy-transport.ts';
20+
import { Buffer } from './node-buffer.ts';
2021
import { getErrorMessage } from './output.ts';
2122
import {
2223
assertNoTransportRedirectInProjectDotenv,
@@ -212,16 +213,48 @@ export function resolveCloudflareD1Context(
212213
};
213214
}
214215

216+
async function withD1SqlFile<T>(sql: string, action: (file: string) => Promise<T>): Promise<T> {
217+
const directory = mkdtempSync(join(tmpdir(), 'void-d1-sql-'));
218+
try {
219+
const file = join(directory, 'input.sql');
220+
writeFileSync(file, sql, { mode: 0o600 });
221+
return await action(file);
222+
} finally {
223+
rmSync(directory, { recursive: true, force: true });
224+
}
225+
}
226+
227+
/** Explicit file imports do not produce query rows. */
228+
export async function importCloudflareD1Sql(
229+
context: CloudflareD1Context,
230+
sql: string,
231+
): Promise<void> {
232+
await withD1SqlFile(sql, (file) =>
233+
run(context.operation, {
234+
type: 'd1.import',
235+
database: context.migrateToken,
236+
file,
237+
configPath: context.operation.configPath,
238+
}),
239+
);
240+
}
241+
215242
export async function executeCloudflareD1Sql(
216243
context: CloudflareD1Context,
217244
sql: string,
218245
): Promise<Array<Record<string, unknown>>> {
219-
const result = await run(context.operation, {
220-
type: 'd1.execute',
221-
database: context.migrateToken,
222-
sql: sql,
223-
configPath: context.operation.configPath,
224-
});
246+
const execute = (source: { sql: string } | { file: string }) =>
247+
run(context.operation, {
248+
type: 'd1.execute',
249+
database: context.migrateToken,
250+
...source,
251+
configPath: context.operation.configPath,
252+
});
253+
// A file changes only the transport, never the query's result semantics.
254+
const result =
255+
Buffer.byteLength(JSON.stringify(sql)) > 16 * 1024
256+
? await withD1SqlFile(sql, (file) => execute({ file }))
257+
: await execute({ sql });
225258
const parsed = requireData(result.data, 'Cloudflare D1 execute');
226259
if (!Array.isArray(parsed)) {
227260
throw new Error('Cloudflare D1 execute returned an unexpected response.');

‎packages/void/src/cli/db.ts‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -933,14 +933,16 @@ async function runExecute(
933933
process.exit(0);
934934
}
935935
if (dialect === 'sqlite') {
936-
const { executeCloudflareD1Sql, resolveCloudflareD1Context } =
936+
const { executeCloudflareD1Sql, importCloudflareD1Sql, resolveCloudflareD1Context } =
937937
await import('./cloudflare-operations.ts');
938-
const rows = await executeCloudflareD1Sql(
939-
resolveCloudflareD1Context(root, configuredD1Binding(root)),
940-
sql,
941-
);
942-
if (rows.length > 0) {
943-
printTable(rows);
938+
const context = resolveCloudflareD1Context(root, configuredD1Binding(root));
939+
if (file) {
940+
await importCloudflareD1Sql(context, sql);
941+
} else {
942+
const rows = await executeCloudflareD1Sql(context, sql);
943+
if (rows.length > 0) {
944+
printTable(rows);
945+
}
944946
}
945947
} else {
946948
const dbUrl = process.env.DATABASE_URL;
Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
import { createHash } from 'node:crypto';
2+
import { writeFileSync } from 'node:fs';
3+
import { join } from 'node:path';
4+
import { afterEach, expect, it, vi } from 'vite-plus/test';
5+
import { importD1SqlFile } from '../../src/cli/cf-d1-import.ts';
6+
import { temporaryDirectory } from '../helpers/cloudflare-operation-process.ts';
7+
8+
const sql = "INSERT INTO messages VALUES ('synthetic');\n";
9+
const etag = createHash('md5').update(sql).digest('hex');
10+
const headers = { Authorization: 'Bearer synthetic-token' };
11+
const endpoint = 'https://api.example.com/import';
12+
13+
function fixture() {
14+
const file = join(temporaryDirectory('void-d1-import-'), 'input.sql');
15+
writeFileSync(file, sql);
16+
return { file, headers, endpoint };
17+
}
18+
function response(result: unknown) {
19+
return Response.json({ success: true, result });
20+
}
21+
22+
afterEach(() => vi.unstubAllGlobals());
23+
24+
it('resumes an already uploaded import without uploading or ingesting twice', async () => {
25+
const fetch = vi
26+
.fn<typeof globalThis.fetch>()
27+
.mockResolvedValueOnce(response({ at_bookmark: 'current', success: true, status: 'active' }))
28+
.mockResolvedValueOnce(response({ status: 'complete', success: true }));
29+
vi.stubGlobal('fetch', fetch);
30+
await importD1SqlFile(fixture());
31+
expect(fetch.mock.calls.map(([, init]) => JSON.parse(String(init?.body)))).toEqual([
32+
{ action: 'init', etag },
33+
{ action: 'poll', current_bookmark: 'current' },
34+
]);
35+
});
36+
37+
it.each([
38+
{
39+
stage: 'initial status',
40+
responses: [response({ status: 'error', error: 'Import unavailable' })],
41+
error: 'Import unavailable',
42+
},
43+
{
44+
stage: 'init',
45+
responses: [
46+
Response.json(
47+
{ success: false, errors: [{ message: 'Permission denied' }] },
48+
{ status: 403 },
49+
),
50+
],
51+
error: 'Permission denied',
52+
},
53+
{
54+
stage: 'upload',
55+
responses: [
56+
response({ filename: 'input.sql', upload_url: 'https://uploads.example.com/sql' }),
57+
new Response('', { headers: { etag: 'wrong' } }),
58+
],
59+
error: 'verification',
60+
},
61+
{
62+
stage: 'ingest',
63+
responses: [
64+
response({ filename: 'input.sql', upload_url: 'https://uploads.example.com/sql' }),
65+
new Response('', { headers: { etag } }),
66+
response({ status: 'error', success: false, error: 'Invalid SQL' }),
67+
],
68+
error: 'Invalid SQL',
69+
},
70+
{
71+
stage: 'poll',
72+
responses: [
73+
response({ at_bookmark: 'current', status: 'active' }),
74+
response({ status: 'error', error: 'Import stopped' }),
75+
],
76+
error: 'Import stopped',
77+
},
78+
{
79+
stage: 'status',
80+
responses: [
81+
response({ at_bookmark: 'current', status: 'active' }),
82+
response({ status: 'complete' }),
83+
],
84+
error: 'invalid D1 import status',
85+
},
86+
])('fails closed at $stage without resubmitting SQL', async ({ responses, error }) => {
87+
const fetch = vi.fn<typeof globalThis.fetch>();
88+
for (const value of responses) {
89+
fetch.mockResolvedValueOnce(value);
90+
}
91+
vi.stubGlobal('fetch', fetch);
92+
await expect(importD1SqlFile(fixture())).rejects.toThrow(error);
93+
expect(fetch).toHaveBeenCalledTimes(responses.length);
94+
});
95+
96+
it('rejects non-HTTPS upload URLs without sending SQL or credentials', async () => {
97+
const fetch = vi.fn<typeof globalThis.fetch>().mockResolvedValueOnce(
98+
response({
99+
filename: 'input.sql',
100+
upload_url: 'http://uploads.example.com/sql',
101+
}),
102+
);
103+
vi.stubGlobal('fetch', fetch);
104+
await expect(importD1SqlFile(fixture())).rejects.toThrow('insecure');
105+
expect(fetch).toHaveBeenCalledTimes(1);
106+
});

0 commit comments

Comments
 (0)