ipfs
diff --git a/‎.github/workflows/interop.yml‎
Lines changed: 32 additions & 8 deletions b/‎.github/workflows/interop.yml‎
Lines changed: 32 additions & 8 deletions
diff --git a/‎config/import.go‎
Lines changed: 95 additions & 17 deletions b/‎config/import.go‎
Lines changed: 95 additions & 17 deletions
diff --git a/‎config/import_test.go‎
Lines changed: 102 additions & 0 deletions b/‎config/import_test.go‎
Lines changed: 102 additions & 0 deletions
@@ -71,18 +71,42 @@ jobs:
           name: kubo
           path: cmd/ipfs
       - run: chmod +x cmd/ipfs/ipfs
-      - run: echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT
-        id: npm-cache-dir
-      - uses: actions/cache@v5
-        with:
-          path: ${{ steps.npm-cache-dir.outputs.dir }}
-          key: ${{ runner.os }}-${{ github.job }}-helia-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: ${{ runner.os }}-${{ github.job }}-helia-
       - run: sudo apt update
       - run: sudo apt install -y libxkbcommon0 libxdamage1 libgbm1 libpango-1.0-0 libcairo2 # dependencies for playwright
-      - run: npx --package @helia/interop helia-interop
+      # Cache node_modules based on latest @helia/interop version from npm registry.
+      # This ensures we always test against the latest release while still benefiting
+      # from caching when the version hasn't changed.
+      - name: Get latest @helia/interop version
+        id: helia-version
+        run: echo "version=$(npm view @helia/interop version)" >> $GITHUB_OUTPUT
+      - name: Cache helia-interop node_modules
+        uses: actions/cache@v5
+        id: helia-cache
+        with:
+          path: node_modules
+          key: ${{ runner.os }}-helia-interop-${{ steps.helia-version.outputs.version }}
+      - name: Install @helia/interop
+        if: steps.helia-cache.outputs.cache-hit != 'true'
+        run: npm install @helia/interop
+      # TODO(IPIP-499): Remove --grep --invert workaround once helia implements IPIP-499
+      # Tracking issue: https://github.com/ipfs/helia/issues/941
+      #
+      # PROVISIONAL HACK: Skip '@helia/mfs - should have the same CID after
+      # creating a file' test due to IPIP-499 changes in kubo.
+      #
+      # WHY IT FAILS: The test creates a 5-byte file in MFS on both kubo and helia,
+      # then compares the root directory CID. With kubo PR #11148, `ipfs files write`
+      # now produces raw CIDs for single-block files (matching `ipfs add --raw-leaves`),
+      # while helia uses `reduceSingleLeafToSelf: false` which keeps the dag-pb wrapper.
+      # Different file CIDs lead to different directory CIDs.
+      #
+      # We run aegir directly (instead of helia-interop binary) because only aegir
+      # supports the --grep/--invert flags needed to exclude specific tests.
+      - name: Run helia-interop tests (excluding IPIP-499 incompatible test)
+        run: npx aegir test -t node --bail -- --grep 'should have the same CID after creating a file' --invert
         env:
           KUBO_BINARY: ${{ github.workspace }}/cmd/ipfs/ipfs
+        working-directory: node_modules/@helia/interop
   ipfs-webui:
     needs: [interop-prep]
     runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }}
 
@@ -2,11 +2,13 @@ package config
 
 import (
 	"fmt"
+	"io"
 	"strconv"
 	"strings"
 
+	chunk "github.com/ipfs/boxo/chunker"
 	"github.com/ipfs/boxo/ipld/unixfs/importer/helpers"
-	"github.com/ipfs/boxo/ipld/unixfs/io"
+	uio "github.com/ipfs/boxo/ipld/unixfs/io"
 	"github.com/ipfs/boxo/verifcid"
 	mh "github.com/multiformats/go-multihash"
 )
@@ -29,29 +31,44 @@ const (
 	// write-batch. The total size of the batch is limited by
 	// BatchMaxnodes and BatchMaxSize.
 	DefaultBatchMaxSize = 100 << 20 // 20MiB
+
+	// HAMTSizeEstimation values for Import.UnixFSHAMTDirectorySizeEstimation
+	HAMTSizeEstimationLinks    = "links"    // legacy: estimate using link names + CID byte lengths (default)
+	HAMTSizeEstimationBlock    = "block"    // full serialized dag-pb block size
+	HAMTSizeEstimationDisabled = "disabled" // disable HAMT sharding entirely
+
+	// DAGLayout values for Import.UnixFSDAGLayout
+	DAGLayoutBalanced = "balanced" // balanced DAG layout (default)
+	DAGLayoutTrickle  = "trickle"  // trickle DAG layout
+
+	DefaultUnixFSHAMTDirectorySizeEstimation = HAMTSizeEstimationLinks // legacy behavior
+	DefaultUnixFSDAGLayout                   = DAGLayoutBalanced       // balanced DAG layout
+	DefaultUnixFSIncludeEmptyDirs            = true                    // include empty directories
 )
 
 var (
 	DefaultUnixFSFileMaxLinks           = int64(helpers.DefaultLinksPerBlock)
 	DefaultUnixFSDirectoryMaxLinks      = int64(0)
-	DefaultUnixFSHAMTDirectoryMaxFanout = int64(io.DefaultShardWidth)
+	DefaultUnixFSHAMTDirectoryMaxFanout = int64(uio.DefaultShardWidth)
 )
 
 // Import configures the default options for ingesting data. This affects commands
 // that ingest data, such as 'ipfs add', 'ipfs dag put, 'ipfs block put', 'ipfs files write'.
 type Import struct {
-	CidVersion                       OptionalInteger
-	UnixFSRawLeaves                  Flag
-	UnixFSChunker                    OptionalString
-	HashFunction                     OptionalString
-	UnixFSFileMaxLinks               OptionalInteger
-	UnixFSDirectoryMaxLinks          OptionalInteger
-	UnixFSHAMTDirectoryMaxFanout     OptionalInteger
-	UnixFSHAMTDirectorySizeThreshold OptionalBytes
-	BatchMaxNodes                    OptionalInteger
-	BatchMaxSize                     OptionalInteger
-	FastProvideRoot                  Flag
-	FastProvideWait                  Flag
+	CidVersion                        OptionalInteger
+	UnixFSRawLeaves                   Flag
+	UnixFSChunker                     OptionalString
+	HashFunction                      OptionalString
+	UnixFSFileMaxLinks                OptionalInteger
+	UnixFSDirectoryMaxLinks           OptionalInteger
+	UnixFSHAMTDirectoryMaxFanout      OptionalInteger
+	UnixFSHAMTDirectorySizeThreshold  OptionalBytes
+	UnixFSHAMTDirectorySizeEstimation OptionalString // "links", "block", or "disabled"
+	UnixFSDAGLayout                   OptionalString // "balanced" or "trickle"
+	BatchMaxNodes                     OptionalInteger
+	BatchMaxSize                      OptionalInteger
+	FastProvideRoot                   Flag
+	FastProvideWait                   Flag
 }
 
 // ValidateImportConfig validates the Import configuration according to UnixFS spec requirements.
@@ -129,6 +146,30 @@ func ValidateImportConfig(cfg *Import) error {
 		}
 	}
 
+	// Validate UnixFSHAMTDirectorySizeEstimation
+	if !cfg.UnixFSHAMTDirectorySizeEstimation.IsDefault() {
+		est := cfg.UnixFSHAMTDirectorySizeEstimation.WithDefault(DefaultUnixFSHAMTDirectorySizeEstimation)
+		switch est {
+		case HAMTSizeEstimationLinks, HAMTSizeEstimationBlock, HAMTSizeEstimationDisabled:
+			// valid
+		default:
+			return fmt.Errorf("Import.UnixFSHAMTDirectorySizeEstimation must be %q, %q, or %q, got %q",
+				HAMTSizeEstimationLinks, HAMTSizeEstimationBlock, HAMTSizeEstimationDisabled, est)
+		}
+	}
+
+	// Validate UnixFSDAGLayout
+	if !cfg.UnixFSDAGLayout.IsDefault() {
+		layout := cfg.UnixFSDAGLayout.WithDefault(DefaultUnixFSDAGLayout)
+		switch layout {
+		case DAGLayoutBalanced, DAGLayoutTrickle:
+			// valid
+		default:
+			return fmt.Errorf("Import.UnixFSDAGLayout must be %q or %q, got %q",
+				DAGLayoutBalanced, DAGLayoutTrickle, layout)
+		}
+	}
+
 	return nil
 }
 
@@ -144,8 +185,7 @@ func isValidChunker(chunker string) bool {
 	}
 
 	// Check for size-<bytes> format
-	if strings.HasPrefix(chunker, "size-") {
-		sizeStr := strings.TrimPrefix(chunker, "size-")
+	if sizeStr, ok := strings.CutPrefix(chunker, "size-"); ok {
 		if sizeStr == "" {
 			return false
 		}
@@ -167,7 +207,7 @@ func isValidChunker(chunker string) bool {
 
 		// Parse and validate min, avg, max values
 		values := make([]int, 3)
-		for i := 0; i < 3; i++ {
+		for i := range 3 {
 			val, err := strconv.Atoi(parts[i+1])
 			if err != nil {
 				return false
@@ -182,3 +222,41 @@ func isValidChunker(chunker string) bool {
 
 	return false
 }
+
+// HAMTSizeEstimationMode returns the boxo SizeEstimationMode based on the config value.
+func (i *Import) HAMTSizeEstimationMode() uio.SizeEstimationMode {
+	switch i.UnixFSHAMTDirectorySizeEstimation.WithDefault(DefaultUnixFSHAMTDirectorySizeEstimation) {
+	case HAMTSizeEstimationLinks:
+		return uio.SizeEstimationLinks
+	case HAMTSizeEstimationBlock:
+		return uio.SizeEstimationBlock
+	case HAMTSizeEstimationDisabled:
+		return uio.SizeEstimationDisabled
+	default:
+		return uio.SizeEstimationLinks
+	}
+}
+
+// UnixFSSplitterFunc returns a SplitterGen function based on Import.UnixFSChunker.
+// The returned function creates a Splitter for the configured chunking strategy.
+// The chunker string is parsed once when this method is called, not on each use.
+func (i *Import) UnixFSSplitterFunc() chunk.SplitterGen {
+	chunkerStr := i.UnixFSChunker.WithDefault(DefaultUnixFSChunker)
+
+	// Parse size-based chunker (most common case) and return optimized generator
+	if sizeStr, ok := strings.CutPrefix(chunkerStr, "size-"); ok {
+		if size, err := strconv.ParseInt(sizeStr, 10, 64); err == nil && size > 0 {
+			return chunk.SizeSplitterGen(size)
+		}
+	}
+
+	// For other chunker types (rabin, buzhash) or invalid config,
+	// fall back to parsing per-use (these are rare cases)
+	return func(r io.Reader) chunk.Splitter {
+		s, err := chunk.FromString(r, chunkerStr)
+		if err != nil {
+			return chunk.DefaultSplitter(r)
+		}
+		return s
+	}
+}
@@ -4,6 +4,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/ipfs/boxo/ipld/unixfs/io"
 	mh "github.com/multiformats/go-multihash"
 )
 
@@ -406,3 +407,104 @@ func TestIsPowerOfTwo(t *testing.T) {
 		})
 	}
 }
+
+func TestValidateImportConfig_HAMTSizeEstimation(t *testing.T) {
+	tests := []struct {
+		name    string
+		value   string
+		wantErr bool
+		errMsg  string
+	}{
+		{name: "valid links", value: HAMTSizeEstimationLinks, wantErr: false},
+		{name: "valid block", value: HAMTSizeEstimationBlock, wantErr: false},
+		{name: "valid disabled", value: HAMTSizeEstimationDisabled, wantErr: false},
+		{name: "invalid unknown", value: "unknown", wantErr: true, errMsg: "must be"},
+		{name: "invalid empty", value: "", wantErr: true, errMsg: "must be"},
+		{name: "invalid typo", value: "link", wantErr: true, errMsg: "must be"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := &Import{
+				UnixFSHAMTDirectorySizeEstimation: *NewOptionalString(tt.value),
+			}
+
+			err := ValidateImportConfig(cfg)
+
+			if tt.wantErr {
+				if err == nil {
+					t.Errorf("expected error for value=%q, got nil", tt.value)
+				} else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) {
+					t.Errorf("error = %v, want error containing %q", err, tt.errMsg)
+				}
+			} else {
+				if err != nil {
+					t.Errorf("unexpected error for value=%q: %v", tt.value, err)
+				}
+			}
+		})
+	}
+}
+
+func TestValidateImportConfig_DAGLayout(t *testing.T) {
+	tests := []struct {
+		name    string
+		value   string
+		wantErr bool
+		errMsg  string
+	}{
+		{name: "valid balanced", value: DAGLayoutBalanced, wantErr: false},
+		{name: "valid trickle", value: DAGLayoutTrickle, wantErr: false},
+		{name: "invalid unknown", value: "unknown", wantErr: true, errMsg: "must be"},
+		{name: "invalid empty", value: "", wantErr: true, errMsg: "must be"},
+		{name: "invalid flat", value: "flat", wantErr: true, errMsg: "must be"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := &Import{
+				UnixFSDAGLayout: *NewOptionalString(tt.value),
+			}
+
+			err := ValidateImportConfig(cfg)
+
+			if tt.wantErr {
+				if err == nil {
+					t.Errorf("expected error for value=%q, got nil", tt.value)
+				} else if tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) {
+					t.Errorf("error = %v, want error containing %q", err, tt.errMsg)
+				}
+			} else {
+				if err != nil {
+					t.Errorf("unexpected error for value=%q: %v", tt.value, err)
+				}
+			}
+		})
+	}
+}
+
+func TestImport_HAMTSizeEstimationMode(t *testing.T) {
+	tests := []struct {
+		cfg  string
+		want io.SizeEstimationMode
+	}{
+		{HAMTSizeEstimationLinks, io.SizeEstimationLinks},
+		{HAMTSizeEstimationBlock, io.SizeEstimationBlock},
+		{HAMTSizeEstimationDisabled, io.SizeEstimationDisabled},
+		{"", io.SizeEstimationLinks},        // default (unset returns default)
+		{"unknown", io.SizeEstimationLinks}, // fallback to default
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.cfg, func(t *testing.T) {
+			var imp Import
+			if tt.cfg != "" {
+				imp.UnixFSHAMTDirectorySizeEstimation = *NewOptionalString(tt.cfg)
+			}
+			got := imp.HAMTSizeEstimationMode()
+			if got != tt.want {
+				t.Errorf("Import.HAMTSizeEstimationMode() with %q = %v, want %v", tt.cfg, got, tt.want)
+			}
+		})
+	}
+}