Skip to content

Commit 1b72824

Browse files
ryancyqNr18
andauthored
fix: binary file corruption (#230)
* fix: image commits leads to invalid images **Location:** `src/blob.ts`, line 28 The code was reading files with UTF-8 text encoding: ```typescript fs.createReadStream(this.absolutePath, { encoding: 'utf8' }) ``` Images are **binary files** containing arbitrary byte values (0-255). The UTF-8 encoding option tells Node.js to: 1. Interpret the binary data as UTF-8 text 2. Drop any byte sequences that are not valid UTF-8 This causes data loss because: - Valid UTF-8 sequences are 1-4 bytes depending on the character - Many byte combinations in binary files are invalid UTF-8 sequences - These invalid bytes are silently dropped by the UTF-8 decoder - The resulting file is corrupted (missing bytes) * fix: prevent binary file corruption when encoding to base64 The base64-encoder was converting base64 strings back to buffers before pushing them to the stream, which caused them to be re-encoded as UTF-8 bytes. This corrupted binary files (images, PDFs, etc.) when they were committed and pushed to GitHub. Changes: - base64-encoder.ts: Push base64 strings directly instead of converting them back to buffers. This prevents the double-encoding that was corrupting binary data. - blob.ts: Remove UTF-8 encoding from file stream reading. Files are now read as raw buffers, which is necessary for the base64 encoder to work correctly with both text and binary files. The fix ensures that: - Binary files (PNG, JPG, PDF, etc.) are encoded without corruption - Text files continue to work correctly - The action properly base64 encodes file content for GitHub API Fixes image corruption when committing and pushing images to GitHub * fix: add binary regression tests to Base64Encoder and Blob.load() using a fixture with non-UTF-8 byte sequences to guard against silent data corruption * chore: update bundle dist --------- Co-authored-by: Joris Conijn <joris@conijnonline.nl>
1 parent 00de5a7 commit 1b72824

7 files changed

Lines changed: 31 additions & 8 deletions

File tree

‎__tests__/blob.test.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,16 @@ describe('Blob', () => {
7272
)
7373
})
7474

75+
it('binary file without corruption', async () => {
76+
const blob = getBlob('fixtures/blob.bin')
77+
const fileAddition = await blob.load()
78+
expect(fileAddition.contents).toEqual(
79+
fs
80+
.readFileSync(join(__dirname, 'fixtures/blob.bin.base64.txt'))
81+
.toString()
82+
)
83+
})
84+
7585
it('file with string', async () => {
7686
const blob = getBlob('fixtures/error.txt')
7787
const mockStream = new PassThrough()

‎__tests__/fixtures/blob.bin‎

12 Bytes
Binary file not shown.
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
iVBORw0KGgr//gAB

‎__tests__/stream/base64-encoder.test.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,18 @@ describe('Base64 Encoder', () => {
1515
const streamedContent = Buffer.concat(chunks).toString('utf8')
1616
expect(streamedContent).toEqual(Buffer.from(content).toString('base64'))
1717
})
18+
19+
it('binary stream without corruption', async () => {
20+
const binary = Buffer.from([
21+
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0xff, 0xfe, 0x00, 0x01,
22+
])
23+
const stream = Readable.from(binary).pipe(new Base64Encoder())
24+
25+
const chunks: Buffer[] = []
26+
for await (const chunk of stream) {
27+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))
28+
}
29+
const streamedContent = Buffer.concat(chunks).toString('utf8')
30+
expect(streamedContent).toEqual(binary.toString('base64'))
31+
})
1832
})

‎dist/index.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23957,12 +23957,12 @@ var Base64Encoder = class extends Transform {
2395723957
chunk = chunk.subarray(0, chunk.length - overflowSize);
2395823958
}
2395923959
const base64String = chunk.toString("base64");
23960-
this.push(Buffer2.from(base64String));
23960+
this.push(base64String);
2396123961
callback();
2396223962
}
2396323963
_flush(callback) {
2396423964
if (this.overflow) {
23965-
this.push(Buffer2.from(this.overflow.toString("base64")));
23965+
this.push(this.overflow.toString("base64"));
2396623966
}
2396723967
callback();
2396823968
}
@@ -23979,7 +23979,7 @@ var Blob2 = class {
2397923979
if (!fs.existsSync(this.absolutePath)) {
2398023980
throw new Error(`File does not exist, path: ${this.absolutePath}`);
2398123981
}
23982-
return fs.createReadStream(this.absolutePath, { encoding: "utf8" }).pipe(new Base64Encoder());
23982+
return fs.createReadStream(this.absolutePath).pipe(new Base64Encoder());
2398323983
}
2398423984
async load() {
2398523985
const chunks = [];

‎src/blob.ts‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,7 @@ export class Blob {
2626
throw new Error(`File does not exist, path: ${this.absolutePath}`)
2727
}
2828

29-
return fs
30-
.createReadStream(this.absolutePath, { encoding: 'utf8' })
31-
.pipe(new Base64Encoder())
29+
return fs.createReadStream(this.absolutePath).pipe(new Base64Encoder())
3230
}
3331

3432
async load(): Promise<FileAddition> {

‎src/stream/base64-encoder.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,13 @@ export default class Base64Encoder extends Transform {
2525
}
2626

2727
const base64String = chunk.toString('base64')
28-
this.push(Buffer.from(base64String))
28+
this.push(base64String)
2929
callback()
3030
}
3131

3232
_flush(callback: TransformCallback): void {
3333
if (this.overflow) {
34-
this.push(Buffer.from(this.overflow.toString('base64')))
34+
this.push(this.overflow.toString('base64'))
3535
}
3636
callback()
3737
}

0 commit comments

Comments
 (0)