fix: make possible to turn off erasure codeing at upload

.github/workflows/tests-rc.yaml

@@ -38,7 +38,7 @@ jobs:
         run: bee-dev --port 16338 --no-swap &
 
       - name: Start bee-factory
-        run: bee-factory start --tag master
+        run: bee-factory start --tag v2.8.2-rc1
 
       - name: Print swarm-cli status
         continue-on-error: true

src/command/manifest/sync.ts

@@ -8,17 +8,10 @@ import { join } from 'path'
 import { pickStamp } from '../../service/stamp'
 import { readdirDeepAsync } from '../../utils'
 import { BzzAddress } from '../../utils/bzz-address'
-import { CommandLineError } from '../../utils/error'
 import { stampProperties } from '../../utils/option'
+import { DEFAULT_REDUNDANCY_LEVEL, determineRedundancyLevel } from '../../utils/redundancy'
 import { RootCommand } from '../root-command'
 
-// Bee applies this level itself whenever a client doesn't ask for a specific one
-// (verified against a real node - bee-js's own docs claim OFF is the server default,
-// which does not match observed behavior). Assumed here so the local pre-check below
-// can replicate the same erasure coding Bee applies, rather than compare against a
-// bare, non-redundant hash that will never match a redundant upload.
-const DEFAULT_REDUNDANCY_LEVEL = RedundancyLevel.MEDIUM
-
 export class Sync extends RootCommand implements LeafCommand {
   public readonly name = 'sync'
   public readonly description = 'Sync a local folder to an existing manifest'
@@ -45,26 +38,6 @@ export class Sync extends RootCommand implements LeafCommand {
   })
   public redundancy!: string
 
-  private determineRedundancyLevel(): RedundancyLevel | undefined {
-    if (!this.redundancy) {
-      return undefined
-    }
-    switch (this.redundancy.toUpperCase()) {
-      case 'OFF':
-        return RedundancyLevel.OFF
-      case 'MEDIUM':
-        return RedundancyLevel.MEDIUM
-      case 'STRONG':
-        return RedundancyLevel.STRONG
-      case 'INSANE':
-        return RedundancyLevel.INSANE
-      case 'PARANOID':
-        return RedundancyLevel.PARANOID
-      default:
-        throw new CommandLineError(`Invalid redundancy level: ${this.redundancy}`)
-    }
-  }
-
   private async expectedReference(data: Uint8Array, level: RedundancyLevel): Promise<Uint8Array> {
     const onBatch = makeErasureBatch(level, false, async () => {
       // no-op: only the resulting hash is needed here, nothing to persist
@@ -87,7 +60,11 @@ export class Sync extends RootCommand implements LeafCommand {
     // node's own implicit default would make the comparison below only as reliable as
     // a guess about that node's behavior, which we've seen differ between Bee versions
     // (2.6.0 defaults to OFF, 2.8.1+ to MEDIUM). Fixing it here removes the ambiguity.
-    const effectiveRedundancyLevel = this.determineRedundancyLevel() ?? DEFAULT_REDUNDANCY_LEVEL
+    const effectiveRedundancyResult = determineRedundancyLevel(this.redundancy)
+    if (effectiveRedundancyResult.error !== undefined) {
+      throw effectiveRedundancyResult.error
+    }
+    const effectiveRedundancyLevel = effectiveRedundancyResult.value ?? DEFAULT_REDUNDANCY_LEVEL
     const uploadOptions = { headers: { 'swarm-redundancy-level': String(effectiveRedundancyLevel) } }
 
     const address = new BzzAddress(this.bzzUrl)

src/command/upload.ts

@@ -1,4 +1,4 @@
-import { FileUploadOptions, RedundancyLevel, Reference, Tag, Utils } from '@ethersphere/bee-js'
+import { BeeRequestOptions, FileUploadOptions, Reference, RedundancyLevel, Tag, Utils } from '@ethersphere/bee-js'
 import { Numbers, Optional, System } from 'cafe-utility'
 import chalk from 'chalk'
 import { Presets, SingleBar } from 'cli-progress'
@@ -16,6 +16,7 @@ import { CommandLineError } from '../utils/error'
 import { getMime } from '../utils/mime'
 import { stampProperties } from '../utils/option'
 import { printQRCodeWithLabel } from '../utils/qr'
+import { determineRedundancyLevel } from '../utils/redundancy'
 import { createSpinner } from '../utils/spinner'
 import { createKeyValue, warningSymbol, warningText } from '../utils/text'
 import { publicUrl } from '../utils/url'
@@ -117,7 +118,7 @@ export class Upload extends RootCommand implements LeafCommand {
 
   @Option({
     key: 'redundancy',
-    description: 'Redundancy of the upload (MEDIUM, STRONG, INSANE, PARANOID)',
+    description: 'Redundancy of the upload (OFF, MEDIUM, STRONG, INSANE, PARANOID)',
   })
   public redundancy!: string
 
@@ -261,15 +262,16 @@ export class Upload extends RootCommand implements LeafCommand {
         encrypt: this.encrypt,
         contentType,
         deferred: this.deferred,
-        redundancyLevel: this.determineRedundancyLevel(),
       } as FileUploadOptions
+
       uploadOptions = this.prepareACTUploadOptions(uploadOptions)
 
       const { reference, historyAddress } = await this.bee.uploadFile(
         this.stamp,
         this.stdinData,
         this.fileName,
         uploadOptions,
+        this.buildRequestOptions(),
       )
       this.result = Optional.of(reference)
 
@@ -283,11 +285,15 @@ export class Upload extends RootCommand implements LeafCommand {
         tag: tag?.uid,
         deferred: this.deferred,
         encrypt: this.encrypt,
-        redundancyLevel: this.determineRedundancyLevel(),
       } as FileUploadOptions
       uploadOptions = this.prepareACTUploadOptions(uploadOptions)
 
-      const { reference, historyAddress } = await this.bee.uploadData(this.stamp, this.stdinData, uploadOptions)
+      const { reference, historyAddress } = await this.bee.uploadData(
+        this.stamp,
+        this.stdinData,
+        uploadOptions,
+        this.buildRequestOptions(),
+      )
       this.result = Optional.of(reference)
 
       if (this.usingACT()) {
@@ -311,10 +317,14 @@ export class Upload extends RootCommand implements LeafCommand {
       pin: this.pin,
       encrypt: this.encrypt,
       deferred: this.deferred,
-      redundancyLevel: this.determineRedundancyLevel(),
     } as FileUploadOptions
     uploadOptions = this.prepareACTUploadOptions(uploadOptions)
-    const { reference, historyAddress } = await this.bee.uploadFilesFromDirectory(this.stamp, this.path, uploadOptions)
+    const { reference, historyAddress } = await this.bee.uploadFilesFromDirectory(
+      this.stamp,
+      this.path,
+      uploadOptions,
+      this.buildRequestOptions(),
+    )
     this.result = Optional.of(reference)
 
     if (this.usingACT()) {
@@ -339,14 +349,14 @@ export class Upload extends RootCommand implements LeafCommand {
       encrypt: this.encrypt,
       contentType,
       deferred: this.deferred,
-      redundancyLevel: this.determineRedundancyLevel(),
     } as FileUploadOptions
     uploadOptions = this.prepareACTUploadOptions(uploadOptions)
     const { reference, historyAddress } = await this.bee.uploadFile(
       this.stamp,
       readable,
       this.determineFileName(parsedPath.base),
       uploadOptions,
+      this.buildRequestOptions(),
     )
     this.result = Optional.of(reference)
 
@@ -417,6 +427,19 @@ export class Upload extends RootCommand implements LeafCommand {
       return
     }
 
+    // Validate here first - Utils.getRedundancyStat below throws its own, less clear
+    // error for anything it doesn't recognize, including OFF, which reaches this method
+    // before buildRequestOptions() ever runs its own validation.
+    const redundancyResult = determineRedundancyLevel(this.redundancy)
+    if (redundancyResult.error !== undefined) {
+      throw redundancyResult.error
+    }
+
+    // Utils.getRedundancyStat only covers the levels that add overhead - OFF has none to report.
+    if (redundancyResult.value === RedundancyLevel.OFF) {
+      return
+    }
+
     const currentSetting = Utils.getRedundancyStat(this.redundancy)
     const originalSize = await this.getUploadSize()
     const originalChunks = Math.ceil(originalSize / 4e3)
@@ -535,22 +558,16 @@ export class Upload extends RootCommand implements LeafCommand {
     return defaultName
   }
 
-  private determineRedundancyLevel(): RedundancyLevel | undefined {
-    if (!this.redundancy) {
-      return undefined
+  private buildRequestOptions(): BeeRequestOptions {
+    const effectiveRedundancyResult = determineRedundancyLevel(this.redundancy)
+    if (effectiveRedundancyResult.error !== undefined) {
+      throw effectiveRedundancyResult.error
     }
-    switch (this.redundancy.toUpperCase()) {
-      case 'MEDIUM':
-        return RedundancyLevel.MEDIUM
-      case 'STRONG':
-        return RedundancyLevel.STRONG
-      case 'INSANE':
-        return RedundancyLevel.INSANE
-      case 'PARANOID':
-        return RedundancyLevel.PARANOID
-      default:
-        throw new CommandLineError(`Invalid redundancy level: ${this.redundancy}`)
+    const uploadHeaders = {} as Record<string, string>
+    if (effectiveRedundancyResult.value !== undefined) {
+      uploadHeaders['swarm-redundancy-level'] = String(effectiveRedundancyResult.value)
     }
+    return { headers: uploadHeaders }
   }
 
   public uploadType(): 'stdin' | 'folder' | 'file' {

src/utils/redundancy.ts

@@ -0,0 +1,29 @@
+import { RedundancyLevel } from '@ethersphere/bee-js'
+import { CommandLineError } from './error'
+
+export const DEFAULT_REDUNDANCY_LEVEL = RedundancyLevel.MEDIUM
+
+export type RedundancyResult = {
+  value?: RedundancyLevel | undefined
+  error?: CommandLineError
+}
+
+export function determineRedundancyLevel(redundancy: string): RedundancyResult {
+  if (!redundancy) {
+    return { value: undefined }
+  }
+  switch (redundancy.toUpperCase()) {
+    case 'OFF':
+      return { value: RedundancyLevel.OFF }
+    case 'MEDIUM':
+      return { value: RedundancyLevel.MEDIUM }
+    case 'STRONG':
+      return { value: RedundancyLevel.STRONG }
+    case 'INSANE':
+      return { value: RedundancyLevel.INSANE }
+    case 'PARANOID':
+      return { value: RedundancyLevel.PARANOID }
+    default:
+      return { error: new CommandLineError(`Invalid redundancy level: ${redundancy}`) }
+  }
+}

test/command/upload.spec.ts

@@ -1,12 +1,35 @@
+import { MerkleTree } from '@ethersphere/bee-js'
 import { System } from 'cafe-utility'
-import { existsSync, unlinkSync, writeFileSync } from 'fs'
+import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'fs'
 import { LeafCommand } from 'furious-commander'
 import QRCode from 'qrcode'
+import { Readable } from 'stream'
 import type { Upload } from '../../src/command/upload'
 import { toBeQRCode, toMatchLinesInOrder } from '../custom-matcher'
 import { describeCommand, invokeTestCli } from '../utility'
 import { getStampOption } from '../utility/stamp'
 
+// process.stdin is a real, singleton stream - once ended via push(null) it can never
+// accept more data, so a test suite that uploads from stdin more than once needs a
+// fresh stream per call rather than reusing the real one.
+async function invokeTestCliWithStdin(argv: string[], data: Buffer): ReturnType<typeof invokeTestCli> {
+  const stream = new Readable({
+    read: () => {
+      // no-op: all data is pushed upfront below, nothing to pull on demand
+    },
+  })
+  stream.push(data)
+  stream.push(null)
+
+  const original = process.stdin
+  Object.defineProperty(process, 'stdin', { value: stream, configurable: true })
+  try {
+    return await invokeTestCli(argv)
+  } finally {
+    Object.defineProperty(process, 'stdin', { value: original, configurable: true })
+  }
+}
+
 const SUCCESSFUL_SYNC_PATTERN = [
   ['Data has been sent to the Bee node successfully!'],
   ['Waiting for file chunks to be synced on Swarm network...'],
@@ -87,6 +110,57 @@ describeCommand(
       })
     })
 
+    describe('redundancy level', () => {
+      // needs to be big enough to span multiple chunks - redundancy only encodes
+      // the intermediate tree, so a single-chunk file would show no difference at all
+      const REDUNDANT_FILE = 'test/testpage/images/swarm.png'
+
+      it('should upload without any redundancy when --redundancy OFF is passed', async () => {
+        const data = readFileSync(REDUNDANT_FILE)
+        const commandBuilder = await invokeTestCliWithStdin(
+          ['upload', '--stdin', '--redundancy', 'OFF', ...getStampOption()],
+          data,
+        )
+        const uploadCommand = commandBuilder.runnable as Upload
+
+        const bareRootChunk = await MerkleTree.root(new Uint8Array(data))
+        const bareReference = Buffer.from(bareRootChunk.hash()).toString('hex')
+
+        expect(uploadCommand.result.getOrThrow().toHex()).toBe(bareReference)
+      })
+
+      it('should produce a different reference than --redundancy OFF when a higher level is requested', async () => {
+        const offBuilder = await invokeTestCli([
+          'upload',
+          REDUNDANT_FILE,
+          '--redundancy',
+          'OFF',
+          '--yes',
+          ...getStampOption(),
+        ])
+        const offReference = (offBuilder.runnable as Upload).result.getOrThrow().toHex()
+
+        // MEDIUM (unlike OFF) prints overhead stats and prompts for confirmation
+        // unless --yes is passed, which would otherwise hang waiting on stdin here.
+        const mediumBuilder = await invokeTestCli([
+          'upload',
+          REDUNDANT_FILE,
+          '--redundancy',
+          'MEDIUM',
+          '--yes',
+          ...getStampOption(),
+        ])
+        const mediumReference = (mediumBuilder.runnable as Upload).result.getOrThrow().toHex()
+
+        expect(mediumReference).not.toBe(offReference)
+      })
+
+      it('should reject an invalid redundancy level', async () => {
+        await invokeTestCli(['upload', 'test/message.txt', '--redundancy', 'NOT_A_LEVEL', ...getStampOption()])
+        expect(hasMessageContaining('Invalid redundancy level')).toBeTruthy()
+      })
+    })
+
     it('should upload folder and encrypt', async () => {
       const commandBuilder = await invokeTestCli(['upload', 'test/testpage', '--encrypt', ...getStampOption()])
       const uploadCommand = commandBuilder.runnable as Upload