From 5a5fa6df78caf944be8f681e0df1632e4cf817bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gergely=20B=C3=A9k=C3=A9si?= Date: Tue, 11 Aug 2026 13:52:40 +0200 Subject: [PATCH 1/2] fix: make possible to turn off erasure codeing at upload --- src/command/manifest/sync.ts | 35 +++-------------- src/command/upload.ts | 61 ++++++++++++++++++----------- src/utils/redundancy.ts | 29 ++++++++++++++ test/command/upload.spec.ts | 76 +++++++++++++++++++++++++++++++++++- 4 files changed, 149 insertions(+), 52 deletions(-) create mode 100644 src/utils/redundancy.ts diff --git a/src/command/manifest/sync.ts b/src/command/manifest/sync.ts index 0c36d90d..ff55b8c2 100644 --- a/src/command/manifest/sync.ts +++ b/src/command/manifest/sync.ts @@ -8,17 +8,10 @@ import { join } from 'path' import { pickStamp } from '../../service/stamp' import { readdirDeepAsync } from '../../utils' import { BzzAddress } from '../../utils/bzz-address' -import { CommandLineError } from '../../utils/error' import { stampProperties } from '../../utils/option' +import { DEFAULT_REDUNDANCY_LEVEL, determineRedundancyLevel } from '../../utils/redundancy' import { RootCommand } from '../root-command' -// Bee applies this level itself whenever a client doesn't ask for a specific one -// (verified against a real node - bee-js's own docs claim OFF is the server default, -// which does not match observed behavior). Assumed here so the local pre-check below -// can replicate the same erasure coding Bee applies, rather than compare against a -// bare, non-redundant hash that will never match a redundant upload. -const DEFAULT_REDUNDANCY_LEVEL = RedundancyLevel.MEDIUM - export class Sync extends RootCommand implements LeafCommand { public readonly name = 'sync' public readonly description = 'Sync a local folder to an existing manifest' @@ -45,26 +38,6 @@ export class Sync extends RootCommand implements LeafCommand { }) public redundancy!: string - private determineRedundancyLevel(): RedundancyLevel | undefined { - if (!this.redundancy) { - return undefined - } - switch (this.redundancy.toUpperCase()) { - case 'OFF': - return RedundancyLevel.OFF - case 'MEDIUM': - return RedundancyLevel.MEDIUM - case 'STRONG': - return RedundancyLevel.STRONG - case 'INSANE': - return RedundancyLevel.INSANE - case 'PARANOID': - return RedundancyLevel.PARANOID - default: - throw new CommandLineError(`Invalid redundancy level: ${this.redundancy}`) - } - } - private async expectedReference(data: Uint8Array, level: RedundancyLevel): Promise { const onBatch = makeErasureBatch(level, false, async () => { // no-op: only the resulting hash is needed here, nothing to persist @@ -87,7 +60,11 @@ export class Sync extends RootCommand implements LeafCommand { // node's own implicit default would make the comparison below only as reliable as // a guess about that node's behavior, which we've seen differ between Bee versions // (2.6.0 defaults to OFF, 2.8.1+ to MEDIUM). Fixing it here removes the ambiguity. - const effectiveRedundancyLevel = this.determineRedundancyLevel() ?? DEFAULT_REDUNDANCY_LEVEL + const effectiveRedundancyResult = determineRedundancyLevel(this.redundancy) + if (effectiveRedundancyResult.error !== undefined) { + throw effectiveRedundancyResult.error + } + const effectiveRedundancyLevel = effectiveRedundancyResult.value ?? DEFAULT_REDUNDANCY_LEVEL const uploadOptions = { headers: { 'swarm-redundancy-level': String(effectiveRedundancyLevel) } } const address = new BzzAddress(this.bzzUrl) diff --git a/src/command/upload.ts b/src/command/upload.ts index 80deeaa1..10e455b1 100644 --- a/src/command/upload.ts +++ b/src/command/upload.ts @@ -1,4 +1,4 @@ -import { FileUploadOptions, RedundancyLevel, Reference, Tag, Utils } from '@ethersphere/bee-js' +import { BeeRequestOptions, FileUploadOptions, Reference, RedundancyLevel, Tag, Utils } from '@ethersphere/bee-js' import { Numbers, Optional, System } from 'cafe-utility' import chalk from 'chalk' import { Presets, SingleBar } from 'cli-progress' @@ -16,6 +16,7 @@ import { CommandLineError } from '../utils/error' import { getMime } from '../utils/mime' import { stampProperties } from '../utils/option' import { printQRCodeWithLabel } from '../utils/qr' +import { determineRedundancyLevel } from '../utils/redundancy' import { createSpinner } from '../utils/spinner' import { createKeyValue, warningSymbol, warningText } from '../utils/text' import { publicUrl } from '../utils/url' @@ -117,7 +118,7 @@ export class Upload extends RootCommand implements LeafCommand { @Option({ key: 'redundancy', - description: 'Redundancy of the upload (MEDIUM, STRONG, INSANE, PARANOID)', + description: 'Redundancy of the upload (OFF, MEDIUM, STRONG, INSANE, PARANOID)', }) public redundancy!: string @@ -261,8 +262,8 @@ export class Upload extends RootCommand implements LeafCommand { encrypt: this.encrypt, contentType, deferred: this.deferred, - redundancyLevel: this.determineRedundancyLevel(), } as FileUploadOptions + uploadOptions = this.prepareACTUploadOptions(uploadOptions) const { reference, historyAddress } = await this.bee.uploadFile( @@ -270,6 +271,7 @@ export class Upload extends RootCommand implements LeafCommand { this.stdinData, this.fileName, uploadOptions, + this.buildRequestOptions(), ) this.result = Optional.of(reference) @@ -283,11 +285,15 @@ export class Upload extends RootCommand implements LeafCommand { tag: tag?.uid, deferred: this.deferred, encrypt: this.encrypt, - redundancyLevel: this.determineRedundancyLevel(), } as FileUploadOptions uploadOptions = this.prepareACTUploadOptions(uploadOptions) - const { reference, historyAddress } = await this.bee.uploadData(this.stamp, this.stdinData, uploadOptions) + const { reference, historyAddress } = await this.bee.uploadData( + this.stamp, + this.stdinData, + uploadOptions, + this.buildRequestOptions(), + ) this.result = Optional.of(reference) if (this.usingACT()) { @@ -311,10 +317,14 @@ export class Upload extends RootCommand implements LeafCommand { pin: this.pin, encrypt: this.encrypt, deferred: this.deferred, - redundancyLevel: this.determineRedundancyLevel(), } as FileUploadOptions uploadOptions = this.prepareACTUploadOptions(uploadOptions) - const { reference, historyAddress } = await this.bee.uploadFilesFromDirectory(this.stamp, this.path, uploadOptions) + const { reference, historyAddress } = await this.bee.uploadFilesFromDirectory( + this.stamp, + this.path, + uploadOptions, + this.buildRequestOptions(), + ) this.result = Optional.of(reference) if (this.usingACT()) { @@ -339,7 +349,6 @@ export class Upload extends RootCommand implements LeafCommand { encrypt: this.encrypt, contentType, deferred: this.deferred, - redundancyLevel: this.determineRedundancyLevel(), } as FileUploadOptions uploadOptions = this.prepareACTUploadOptions(uploadOptions) const { reference, historyAddress } = await this.bee.uploadFile( @@ -347,6 +356,7 @@ export class Upload extends RootCommand implements LeafCommand { readable, this.determineFileName(parsedPath.base), uploadOptions, + this.buildRequestOptions(), ) this.result = Optional.of(reference) @@ -417,6 +427,19 @@ export class Upload extends RootCommand implements LeafCommand { return } + // Validate here first - Utils.getRedundancyStat below throws its own, less clear + // error for anything it doesn't recognize, including OFF, which reaches this method + // before buildRequestOptions() ever runs its own validation. + const redundancyResult = determineRedundancyLevel(this.redundancy) + if (redundancyResult.error !== undefined) { + throw redundancyResult.error + } + + // Utils.getRedundancyStat only covers the levels that add overhead - OFF has none to report. + if (redundancyResult.value === RedundancyLevel.OFF) { + return + } + const currentSetting = Utils.getRedundancyStat(this.redundancy) const originalSize = await this.getUploadSize() const originalChunks = Math.ceil(originalSize / 4e3) @@ -535,22 +558,16 @@ export class Upload extends RootCommand implements LeafCommand { return defaultName } - private determineRedundancyLevel(): RedundancyLevel | undefined { - if (!this.redundancy) { - return undefined + private buildRequestOptions(): BeeRequestOptions { + const effectiveRedundancyResult = determineRedundancyLevel(this.redundancy) + if (effectiveRedundancyResult.error !== undefined) { + throw effectiveRedundancyResult.error } - switch (this.redundancy.toUpperCase()) { - case 'MEDIUM': - return RedundancyLevel.MEDIUM - case 'STRONG': - return RedundancyLevel.STRONG - case 'INSANE': - return RedundancyLevel.INSANE - case 'PARANOID': - return RedundancyLevel.PARANOID - default: - throw new CommandLineError(`Invalid redundancy level: ${this.redundancy}`) + const uploadHeaders = {} as Record + if (effectiveRedundancyResult.value !== undefined) { + uploadHeaders['swarm-redundancy-level'] = String(effectiveRedundancyResult.value) } + return { headers: uploadHeaders } } public uploadType(): 'stdin' | 'folder' | 'file' { diff --git a/src/utils/redundancy.ts b/src/utils/redundancy.ts new file mode 100644 index 00000000..99db50dd --- /dev/null +++ b/src/utils/redundancy.ts @@ -0,0 +1,29 @@ +import { RedundancyLevel } from '@ethersphere/bee-js' +import { CommandLineError } from './error' + +export const DEFAULT_REDUNDANCY_LEVEL = RedundancyLevel.MEDIUM + +export type RedundancyResult = { + value?: RedundancyLevel | undefined + error?: CommandLineError +} + +export function determineRedundancyLevel(redundancy: string): RedundancyResult { + if (!redundancy) { + return { value: undefined } + } + switch (redundancy.toUpperCase()) { + case 'OFF': + return { value: RedundancyLevel.OFF } + case 'MEDIUM': + return { value: RedundancyLevel.MEDIUM } + case 'STRONG': + return { value: RedundancyLevel.STRONG } + case 'INSANE': + return { value: RedundancyLevel.INSANE } + case 'PARANOID': + return { value: RedundancyLevel.PARANOID } + default: + return { error: new CommandLineError(`Invalid redundancy level: ${redundancy}`) } + } +} diff --git a/test/command/upload.spec.ts b/test/command/upload.spec.ts index d15e768e..4f9ecb63 100644 --- a/test/command/upload.spec.ts +++ b/test/command/upload.spec.ts @@ -1,12 +1,35 @@ +import { MerkleTree } from '@ethersphere/bee-js' import { System } from 'cafe-utility' -import { existsSync, unlinkSync, writeFileSync } from 'fs' +import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'fs' import { LeafCommand } from 'furious-commander' import QRCode from 'qrcode' +import { Readable } from 'stream' import type { Upload } from '../../src/command/upload' import { toBeQRCode, toMatchLinesInOrder } from '../custom-matcher' import { describeCommand, invokeTestCli } from '../utility' import { getStampOption } from '../utility/stamp' +// process.stdin is a real, singleton stream - once ended via push(null) it can never +// accept more data, so a test suite that uploads from stdin more than once needs a +// fresh stream per call rather than reusing the real one. +async function invokeTestCliWithStdin(argv: string[], data: Buffer): ReturnType { + const stream = new Readable({ + read: () => { + // no-op: all data is pushed upfront below, nothing to pull on demand + }, + }) + stream.push(data) + stream.push(null) + + const original = process.stdin + Object.defineProperty(process, 'stdin', { value: stream, configurable: true }) + try { + return await invokeTestCli(argv) + } finally { + Object.defineProperty(process, 'stdin', { value: original, configurable: true }) + } +} + const SUCCESSFUL_SYNC_PATTERN = [ ['Data has been sent to the Bee node successfully!'], ['Waiting for file chunks to be synced on Swarm network...'], @@ -87,6 +110,57 @@ describeCommand( }) }) + describe('redundancy level', () => { + // needs to be big enough to span multiple chunks - redundancy only encodes + // the intermediate tree, so a single-chunk file would show no difference at all + const REDUNDANT_FILE = 'test/testpage/images/swarm.png' + + it('should upload without any redundancy when --redundancy OFF is passed', async () => { + const data = readFileSync(REDUNDANT_FILE) + const commandBuilder = await invokeTestCliWithStdin( + ['upload', '--stdin', '--redundancy', 'OFF', ...getStampOption()], + data, + ) + const uploadCommand = commandBuilder.runnable as Upload + + const bareRootChunk = await MerkleTree.root(new Uint8Array(data)) + const bareReference = Buffer.from(bareRootChunk.hash()).toString('hex') + + expect(uploadCommand.result.getOrThrow().toHex()).toBe(bareReference) + }) + + it('should produce a different reference than --redundancy OFF when a higher level is requested', async () => { + const offBuilder = await invokeTestCli([ + 'upload', + REDUNDANT_FILE, + '--redundancy', + 'OFF', + '--yes', + ...getStampOption(), + ]) + const offReference = (offBuilder.runnable as Upload).result.getOrThrow().toHex() + + // MEDIUM (unlike OFF) prints overhead stats and prompts for confirmation + // unless --yes is passed, which would otherwise hang waiting on stdin here. + const mediumBuilder = await invokeTestCli([ + 'upload', + REDUNDANT_FILE, + '--redundancy', + 'MEDIUM', + '--yes', + ...getStampOption(), + ]) + const mediumReference = (mediumBuilder.runnable as Upload).result.getOrThrow().toHex() + + expect(mediumReference).not.toBe(offReference) + }) + + it('should reject an invalid redundancy level', async () => { + await invokeTestCli(['upload', 'test/message.txt', '--redundancy', 'NOT_A_LEVEL', ...getStampOption()]) + expect(hasMessageContaining('Invalid redundancy level')).toBeTruthy() + }) + }) + it('should upload folder and encrypt', async () => { const commandBuilder = await invokeTestCli(['upload', 'test/testpage', '--encrypt', ...getStampOption()]) const uploadCommand = commandBuilder.runnable as Upload From ff025ba4600732ad7651fcc593e048eec5cfba4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gergely=20B=C3=A9k=C3=A9si?= Date: Tue, 11 Aug 2026 14:20:54 +0200 Subject: [PATCH 2/2] chore: use latest rc --- .github/workflows/tests-rc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-rc.yaml b/.github/workflows/tests-rc.yaml index 5393d352..4ae1f60f 100644 --- a/.github/workflows/tests-rc.yaml +++ b/.github/workflows/tests-rc.yaml @@ -38,7 +38,7 @@ jobs: run: bee-dev --port 16338 --no-swap & - name: Start bee-factory - run: bee-factory start --tag master + run: bee-factory start --tag v2.8.2-rc1 - name: Print swarm-cli status continue-on-error: true