diff --git a/packages/flutter/lib/src/material/scaffold.dart b/packages/flutter/lib/src/material/scaffold.dart
index 01d4759..e7aae7d 100644
--- a/packages/flutter/lib/src/material/scaffold.dart
+++ b/packages/flutter/lib/src/material/scaffold.dart
@@ -2177,6 +2177,7 @@
   final List<_StandardBottomSheet> _dismissedBottomSheets = <_StandardBottomSheet>[];
   PersistentBottomSheetController<dynamic>? _currentBottomSheet;
   final GlobalKey _currentBottomSheetKey = GlobalKey();
+  LocalHistoryEntry? _persistentSheetHistoryEntry;
 
   void _maybeBuildPersistentBottomSheet() {
     if (widget.bottomSheet != null && _currentBottomSheet == null) {
@@ -2184,22 +2185,19 @@
       // will not be added to the Scaffold's appbar and the bottom sheet will not
       // support drag or swipe to dismiss.
       final AnimationController animationController = BottomSheet.createAnimationController(this)..value = 1.0;
-      LocalHistoryEntry? persistentSheetHistoryEntry;
       bool persistentBottomSheetExtentChanged(DraggableScrollableNotification notification) {
-        if (notification.extent > notification.initialExtent) {
-          if (persistentSheetHistoryEntry == null) {
-            persistentSheetHistoryEntry = LocalHistoryEntry(onRemove: () {
-              if (notification.extent > notification.initialExtent) {
-                DraggableScrollableActuator.reset(notification.context);
-              }
+        if (notification.extent - notification.initialExtent > precisionErrorTolerance) {
+          if (_persistentSheetHistoryEntry == null) {
+            _persistentSheetHistoryEntry = LocalHistoryEntry(onRemove: () {
+              DraggableScrollableActuator.reset(notification.context);
               showBodyScrim(false, 0.0);
               _floatingActionButtonVisibilityValue = 1.0;
-              persistentSheetHistoryEntry = null;
+              _persistentSheetHistoryEntry = null;
             });
-            ModalRoute.of(context)!.addLocalHistoryEntry(persistentSheetHistoryEntry!);
+            ModalRoute.of(context)!.addLocalHistoryEntry(_persistentSheetHistoryEntry!);
           }
-        } else if (persistentSheetHistoryEntry != null) {
-          ModalRoute.of(context)!.removeLocalHistoryEntry(persistentSheetHistoryEntry!);
+        } else if (_persistentSheetHistoryEntry != null) {
+          _persistentSheetHistoryEntry!.remove();
         }
         return false;
       }
@@ -2298,6 +2296,15 @@
 
     bool removedEntry = false;
     bool doingDispose = false;
+
+    void removePersistentSheetHistoryEntryIfNeeded() {
+      assert(isPersistent);
+      if (_persistentSheetHistoryEntry != null) {
+        _persistentSheetHistoryEntry!.remove();
+        _persistentSheetHistoryEntry = null;
+      }
+    }
+
     void removeCurrentBottomSheet() {
       removedEntry = true;
       if (_currentBottomSheet == null) {
@@ -2307,6 +2314,10 @@
       assert(bottomSheetKey.currentState != null);
       _showFloatingActionButton();
 
+      if (isPersistent) {
+        removePersistentSheetHistoryEntryIfNeeded();
+      }
+
       bottomSheetKey.currentState!.close();
       setState(() {
         _currentBottomSheet = null;
diff --git a/packages/flutter/test/material/persistent_bottom_sheet_test.dart b/packages/flutter/test/material/persistent_bottom_sheet_test.dart
index 17302b3..7aa890a 100644
--- a/packages/flutter/test/material/persistent_bottom_sheet_test.dart
+++ b/packages/flutter/test/material/persistent_bottom_sheet_test.dart
@@ -22,6 +22,61 @@
     expect(dyDelta1, isNot(moreOrLessEquals(dyDelta2, epsilon: 0.1)));
   }
 
+  testWidgets('Persistent draggableScrollableSheet localHistoryEntries test', (WidgetTester tester) async {
+    // Regression test for https://github.com/flutter/flutter/issues/110123
+    Widget buildFrame(Widget? bottomSheet) {
+      return MaterialApp(
+        home: Scaffold(
+          appBar: AppBar(),
+          body: const Center(child: Text('body')),
+          bottomSheet: bottomSheet,
+          floatingActionButton: const FloatingActionButton(
+            onPressed: null,
+            child: Text('fab'),
+          ),
+        ),
+      );
+    }
+    final Widget draggableScrollableSheet = DraggableScrollableSheet(
+      expand: false,
+      snap: true,
+      initialChildSize: 0.3,
+      minChildSize: 0.3,
+      builder: (_, ScrollController controller) {
+        return ListView.builder(
+          itemExtent: 50.0,
+          itemCount: 50,
+          itemBuilder: (_, int index) => Text('Item $index'),
+          controller: controller,
+        );
+      },
+    );
+
+    await tester.pumpWidget(buildFrame(draggableScrollableSheet));
+    await tester.pumpAndSettle();
+
+    expect(find.byType(BackButton).hitTestable(), findsNothing);
+
+    await tester.drag(find.text('Item 2'), const Offset(0, -200.0));
+    await tester.pumpAndSettle();
+    // We've started to drag up, we should have a back button now for a11y
+    expect(find.byType(BackButton).hitTestable(), findsOneWidget);
+
+    await tester.fling(find.text('Item 2'), const Offset(0, 200.0), 2000.0);
+    await tester.pumpAndSettle();
+    // BackButton should be hidden
+    expect(find.byType(BackButton).hitTestable(), findsNothing);
+
+    // Show the back button again
+    await tester.drag(find.text('Item 2'), const Offset(0, -200.0));
+    await tester.pumpAndSettle();
+    expect(find.byType(BackButton).hitTestable(), findsOneWidget);
+
+    // Remove the draggableScrollableSheet should hide the back button
+    await tester.pumpWidget(buildFrame(null));
+    expect(find.byType(BackButton).hitTestable(), findsNothing);
+  });
+
   // Regression test for https://github.com/flutter/flutter/issues/83668
   testWidgets('Scaffold.bottomSheet update test', (WidgetTester tester) async {
     Widget buildFrame(Widget? bottomSheet) {