feat(caching): implement negative stat cache to optimize polling of missing files

cmd/mount.go

@@ -108,6 +108,7 @@ be interacting with the file system.`)
 		DummyIOCfg:                         newConfig.DummyIo,
 		IsTypeCacheDeprecated:              newConfig.EnableTypeCacheDeprecation,
 		ImplicitDir:                        newConfig.ImplicitDirs,
+		EnableEmptyManagedFolders:          newConfig.List.EnableEmptyManagedFolders,
 	}
 	bm := gcsx.NewBucketManager(bucketCfg, storageHandle)
 

docs/semantics.md

@@ -209,7 +209,12 @@ The behavior of stat cache is controlled by the following flags/config parameter
    If this config variable is missing, then the value of ```--stat-cache-ttl``` is used.
    * ```--stat-cache-ttl``` commandline flag, which can be set to a value like ```10s``` or ```1.5h```. The default is one minute. This has been deprecated (starting v2.0) and is currently only available for backward compatibility. If ```metadata-cache: ttl-secs``` is set, ```--stat-cache-ttl``` is ignored.
 
-   Positive and negative stat results will be cached for the specified amount of time.
+   Positive stat results (existing files) will be cached for this specified amount of time.
+
+3. **Negative Stat-cache (Non-existent Entry Caching)**: Controls caching of non-existent paths (`ErrNotExist` / 404 results) to optimize workloads that aggressively poll missing files (e.g., JupyterLab).
+   * `metadata-cache: negative-ttl-secs` in the config-file (integer). Sets the TTL in seconds for negative entries. Default is 5 seconds. Use -1 for infinite TTL. Setting this to 0 disables negative caching.
+      > [!WARNING]
+      > Using an infinite TTL (`-1`) for negative caching is risky if objects are created externally (by other clients or processes), as those new files/directories will never be visible under the mount until the cache is manually cleared or GCSFuse is restarted.
 
 Warnings: 
 - Using stat caching breaks the consistency guarantees discussed in this document. It is safe only in the following situations:

internal/fs/caching_test.go

@@ -69,6 +69,7 @@ func (t *cachingTestCommon) SetUpTestSuite() {
 		negativeCacheTTL,
 		IsTypeCacheDeprecated,
 		isImplicitDir,
+		false,
 	)
 
 	// Enable directory type caching.
@@ -338,6 +339,9 @@ func (t *CachingWithImplicitDirsTest) ImplicitDirectory_DefinedByFile() {
 
 	AssertEq(nil, err)
 
+	// Advance time so that any negative cache for "foo" from previous tests expires.
+	cacheClock.AdvanceTime(negativeCacheTTL + time.Millisecond)
+
 	// The directory should appear to exist.
 	fi, err = os.Stat(path.Join(mntDir, "foo"))
 	AssertEq(nil, err)
@@ -359,6 +363,9 @@ func (t *CachingWithImplicitDirsTest) ImplicitDirectory_DefinedByDirectory() {
 
 	AssertEq(nil, err)
 
+	// Advance time so that any negative cache for "foo" from previous tests expires.
+	cacheClock.AdvanceTime(negativeCacheTTL + time.Millisecond)
+
 	// The directory should appear to exist.
 	fi, err = os.Stat(path.Join(mntDir, "foo"))
 	AssertEq(nil, err)
@@ -479,6 +486,7 @@ func (t *MultiBucketMountCachingTest) SetUpTestSuite() {
 			negativeCacheTTL,
 			IsTypeCacheDeprecated,
 			isImplicitDir,
+			false,
 		)
 	}
 

internal/fs/gcs_metrics_test.go

@@ -20,11 +20,16 @@ import (
 	"os"
 	"testing"
 
+	"time"
+
 	"github.com/googlecloudplatform/gcsfuse/v3/cfg"
+	"github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru"
+	"github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/fs"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/fs/wrappers"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/monitor"
+	"github.com/googlecloudplatform/gcsfuse/v3/internal/storage/caching"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/storage/fake"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/storage/gcs"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/storage/storageutil"
@@ -463,3 +468,89 @@ func TestGCSMetrics_RetryCount(t *testing.T) {
 		attribute.NewSet(attribute.String("retry_error_category", "STALLED_READ_REQUEST")),
 		1)
 }
+
+type fakeBucketManagerForShortCircuit struct {
+	bucket  gcs.Bucket
+	tempDir string
+}
+
+func (bm *fakeBucketManagerForShortCircuit) SetUpBucket(ctx context.Context, name string, _ bool, _ metrics.MetricHandle) (gcsx.SyncerBucket, error) {
+	return gcsx.NewSyncerBucket(0, 120, 10, bm.tempDir, gcsx.NewContentTypeBucket(bm.bucket)), nil
+}
+func (bm *fakeBucketManagerForShortCircuit) ShutDown() {}
+
+// TestGCSMetrics_RequestCount_NegativeCachingShortCircuit validates that when negative entry caching is enabled,
+// repeated lookups for non-existent files short-circuit in memory and do not emit redundant backend GCS requests.
+func TestGCSMetrics_RequestCount_NegativeCachingShortCircuit(t *testing.T) {
+	ctx := context.Background()
+	origProvider := otel.GetMeterProvider()
+	t.Cleanup(func() { otel.SetMeterProvider(origProvider) })
+	reader := metric.NewManualReader()
+	provider := metric.NewMeterProvider(metric.WithReader(reader))
+	otel.SetMeterProvider(provider)
+	mh, err := metrics.NewOTelMetrics(ctx, 1, 100)
+	require.NoError(t, err, "metrics.NewOTelMetrics")
+	bucketName := "test-bucket"
+	rawBucket := fake.NewFakeBucket(timeutil.RealClock(), bucketName, gcs.BucketType{Hierarchical: false})
+	monitoringInnerBucket := monitor.NewMonitoringBucket(rawBucket, mh)
+	lruCache := lru.NewCache(1024 * 1024)
+	statCacheView := metadata.NewStatCacheBucketView(lruCache, "")
+	cachedOuterBucket := caching.NewFastStatBucket(
+		time.Minute,
+		statCacheView,
+		timeutil.RealClock(),
+		monitoringInnerBucket,
+		time.Minute,
+		true, // isTypeCacheDeprecated
+		true, // implicitDir
+		false,
+	)
+	serverCfg := &fs.ServerConfig{
+		NewConfig: &cfg.Config{
+			EnableNewReader:            true,
+			EnableTypeCacheDeprecation: true,
+			MetadataCache: cfg.MetadataCacheConfig{
+				TtlSecs: 60,
+			},
+		},
+		MetricHandle: mh,
+		TraceHandle:  tracing.NewNoopTracer(),
+		CacheClock:   &timeutil.SimulatedClock{},
+		BucketName:   bucketName,
+		BucketManager: &fakeBucketManagerForShortCircuit{
+			bucket:  cachedOuterBucket,
+			tempDir: t.TempDir(),
+		},
+		ImplicitDirectories: true,
+	}
+	server, err := fs.NewFileSystem(ctx, serverCfg)
+	require.NoError(t, err, "NewFileSystem")
+	lookupOp := &fuseops.LookUpInodeOp{
+		Parent: fuseops.RootInodeID,
+		Name:   "missing_file",
+	}
+
+	// First Lookup - Cache Miss
+	_ = server.LookUpInode(ctx, lookupOp)
+	waitForMetricsProcessing()
+
+	// Probing a missing file checks file (StatObject) and then implicit dir (ListObjects)
+	metrics.VerifyCounterMetric(t, ctx, reader, "gcs/request_count",
+		attribute.NewSet(attribute.String("gcs_method", "StatObject")),
+		1)
+	metrics.VerifyCounterMetric(t, ctx, reader, "gcs/request_count",
+		attribute.NewSet(attribute.String("gcs_method", "ListObjects")),
+		1)
+
+	// Second Lookup - Negative Cache Hit
+	_ = server.LookUpInode(ctx, lookupOp)
+	waitForMetricsProcessing()
+
+	// StatObject is short-circuited (remains 1), and ListObjects is also short-circuited (remains 1)
+	metrics.VerifyCounterMetric(t, ctx, reader, "gcs/request_count",
+		attribute.NewSet(attribute.String("gcs_method", "StatObject")),
+		1)
+	metrics.VerifyCounterMetric(t, ctx, reader, "gcs/request_count",
+		attribute.NewSet(attribute.String("gcs_method", "ListObjects")),
+		1)
+}

internal/fs/hns_bucket_test.go

@@ -172,7 +172,8 @@ func (t *HNSCachedBucketMountTest) SetupSuite() {
 		uncachedHNSBucket,
 		negativeCacheTTL,
 		IsTypeCacheDeprecated,
-		isImplicitDir)
+		isImplicitDir,
+		false)
 
 	// Enable directory type caching.
 	t.serverCfg.DirTypeCacheTTL = ttl

internal/fs/inode/dir.go

@@ -683,31 +683,38 @@ func (d *dirInode) LookUpChild(ctx context.Context, name string) (*Core, error)
 
 		// 1. Try Directory FIRST (since it's the preferred return type)
 		var dirResult *Core
-		var err error
+		var dirErr error
 		if d.Bucket().BucketType().Hierarchical {
-			dirResult, err = findExplicitFolder(ctx, d.Bucket(), NewDirName(d.Name(), name), true)
+			dirResult, dirErr = findExplicitFolder(ctx, d.Bucket(), NewDirName(d.Name(), name), true)
 		} else {
-			dirResult, err = findExplicitInode(ctx, d.Bucket(), NewDirName(d.Name(), name), true)
+			dirResult, dirErr = findExplicitInode(ctx, d.Bucket(), NewDirName(d.Name(), name), true)
+		}
+
+		// If we hit a real error (not a cache miss), exit early.
+		if dirErr != nil && !errors.As(dirErr, &cacheMissErr) {
+			return nil, dirErr
 		}
 
 		// If we found a directory, we're done. Return it now.
 		if dirResult != nil {
 			return dirResult, nil
 		}
-		// If we hit a real error (not a cache miss), exit early.
-		if err != nil && !errors.As(err, &cacheMissErr) {
-			return nil, err
-		}
 
 		// 2. Try File ONLY if directory wasn't found
-		fileResult, err := findExplicitInode(ctx, d.Bucket(), NewFileName(d.Name(), name), true)
-		if err != nil && !errors.As(err, &cacheMissErr) {
-			return nil, err
+		fileResult, fileErr := findExplicitInode(ctx, d.Bucket(), NewFileName(d.Name(), name), true)
+		if fileErr != nil && !errors.As(fileErr, &cacheMissErr) {
+			return nil, fileErr
 		}
 
 		if fileResult != nil {
 			return fileResult, nil
 		}
+
+		// 3. Both lookups resulted in cache hits (no cacheMiss errors) with no results found,
+		// indicating a negative cache entry. Return nil to indicate the entry doesn't exist from cache.
+		if dirErr == nil && fileErr == nil {
+			return nil, nil
+		}
 	}
 
 	// 2. Legacy: If Type Cache is NOT deprecated, fetch the type hint.

internal/fs/inode/dir_test.go

@@ -24,6 +24,8 @@ import (
 	"testing"
 	"time"
 
+	"context"
+
 	"github.com/googlecloudplatform/gcsfuse/v3/cfg"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/cache/metadata"
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/contentcache"
@@ -39,7 +41,6 @@ import (
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
-	"golang.org/x/net/context"
 	"golang.org/x/sync/semaphore"
 
 	"github.com/googlecloudplatform/gcsfuse/v3/internal/gcsx"
@@ -348,6 +349,54 @@ func (t *DirTest) TestLookUpChild_NonExistent() {
 	}
 }
 
+type mockNegativeCacheBucket struct {
+	gcs.Bucket
+}
+
+func (m *mockNegativeCacheBucket) StatObject(ctx context.Context, req *gcs.StatObjectRequest) (*gcs.MinObject, *gcs.ExtendedObjectAttributes, error) {
+	return nil, nil, &gcs.NotFoundError{Err: errors.New("negative cache")}
+}
+
+func (m *mockNegativeCacheBucket) BucketType() gcs.BucketType {
+	return gcs.BucketType{}
+}
+
+func (t *DirTest) TestLookUpChild_NegativeCacheHit() {
+	const name = "qux"
+
+	mockBucket := &mockNegativeCacheBucket{}
+	syncerBucket := gcsx.NewSyncerBucket(
+		1,
+		chunkRetryDeadlineSecs,
+		chunkTransferTimeoutSecs,
+		".gcsfuse_tmp/",
+		mockBucket)
+
+	config := &cfg.Config{
+		EnableTypeCacheDeprecation: true,
+	}
+
+	in := NewDirInode(
+		dirInodeID,
+		NewDirName(NewRootName(""), dirInodeName),
+		context.Background(),
+		fuseops.InodeAttributes{},
+		false,
+		false,
+		time.Second, // non-zero TTL to enable the caching logic
+		&syncerBucket,
+		&t.clock,
+		&t.clock,
+		semaphore.NewWeighted(10),
+		config,
+	)
+
+	result, err := in.LookUpChild(t.ctx, name)
+
+	require.NoError(t.T(), err)
+	require.Nil(t.T(), result)
+}
+
 func (t *DirTest) TestLookUpChild_FileOnly() {
 	const name = "qux"
 	objName := path.Join(dirInodeName, name)

internal/gcsx/bucket_manager.go

@@ -78,6 +78,8 @@ type BucketConfig struct {
 	IsTypeCacheDeprecated bool
 
 	ImplicitDir bool
+
+	EnableEmptyManagedFolders bool
 }
 
 // BucketManager manages the lifecycle of buckets.
@@ -241,7 +243,8 @@ func (bm *bucketManager) SetUpBucket(
 			b,
 			bm.config.NegativeStatCacheTTL,
 			bm.config.IsTypeCacheDeprecated,
-			bm.config.ImplicitDir)
+			bm.config.ImplicitDir,
+			bm.config.EnableEmptyManagedFolders)
 	}
 
 	// Enable content type awareness

internal/storage/caching/fast_stat_bucket.go

@@ -51,15 +51,17 @@ func NewFastStatBucket(
 	negativeCacheTTL time.Duration,
 	isTypeCacheDeprecated bool,
 	implicitDir bool,
+	enableEmptyManagedFolders bool,
 ) (b gcs.Bucket) {
 	fsb := &fastStatBucket{
-		cache:                 cache,
-		clock:                 clock,
-		wrapped:               wrapped,
-		primaryCacheTTL:       primaryCacheTTL,
-		negativeCacheTTL:      negativeCacheTTL,
-		isTypeCacheDeprecated: isTypeCacheDeprecated,
-		implicitDir:           implicitDir,
+		cache:                     cache,
+		clock:                     clock,
+		wrapped:                   wrapped,
+		primaryCacheTTL:           primaryCacheTTL,
+		negativeCacheTTL:          negativeCacheTTL,
+		isTypeCacheDeprecated:     isTypeCacheDeprecated,
+		implicitDir:               implicitDir,
+		enableEmptyManagedFolders: enableEmptyManagedFolders,
 	}
 
 	b = fsb
@@ -92,6 +94,8 @@ type fastStatBucket struct {
 	isTypeCacheDeprecated bool
 
 	implicitDir bool
+
+	enableEmptyManagedFolders bool
 }
 
 ////////////////////////////////////////////////////////////////////////
@@ -145,6 +149,14 @@ func (b *fastStatBucket) insertListing(ctx context.Context, listing *gcs.Listing
 	if !b.implicitDir {
 		return
 	}
+	// Negative Cache (Only if it's a directory and there are NO contents)
+	// If enableEmptyManagedFolders is true, do not negatively cache empty directories,
+	// otherwise existing positive entries for valid empty managed folders get overwritten.
+	isNegativeCacheEnabled := b.negativeCacheTTL > 0 && !b.enableEmptyManagedFolders
+	isEmptyNonRootDir := !dirHasContents && dirName != ""
+	if isNegativeCacheEnabled && isEmptyNonRootDir {
+		b.cache.AddNegativeEntry(dirName, b.clock.Now().Add(b.negativeCacheTTL))
+	}
 
 	// 3. Cache Sub-directories (Collapsed Runs)
 	// These represent folders discovered via prefixes in the ListObjects response.