diff --git a/packages/flutter/lib/src/widgets/scrollbar.dart b/packages/flutter/lib/src/widgets/scrollbar.dart
index 00cd2a7..15b3874 100644
--- a/packages/flutter/lib/src/widgets/scrollbar.dart
+++ b/packages/flutter/lib/src/widgets/scrollbar.dart
@@ -1447,6 +1447,7 @@
 /// scrollbar track.
 class RawScrollbarState<T extends RawScrollbar> extends State<T> with TickerProviderStateMixin<T> {
   Offset? _dragScrollbarAxisOffset;
+  late double? _thumbPress;
   ScrollController? _currentController;
   Timer? _fadeoutTimer;
   late AnimationController _fadeoutAnimationController;
@@ -1785,6 +1786,9 @@
     _fadeoutTimer?.cancel();
     _fadeoutAnimationController.forward();
     _dragScrollbarAxisOffset = localPosition;
+    _thumbPress = direction == Axis.vertical
+      ? localPosition.dy - scrollbarPainter._thumbOffset
+      : localPosition.dx - scrollbarPainter._thumbOffset;
   }
 
   /// Handler called when a currently active long press gesture moves.
@@ -1802,10 +1806,28 @@
     if (direction == null) {
       return;
     }
-    _updateScrollPosition(localPosition);
+    switch (position.axisDirection) {
+      case AxisDirection.up:
+      case AxisDirection.down:
+        if (_canDragThumb(_dragScrollbarAxisOffset!.dy, position.viewportDimension, _thumbPress!)) {
+          _updateScrollPosition(localPosition);
+        }
+        break;
+      case AxisDirection.left:
+      case AxisDirection.right:
+        if (_canDragThumb(_dragScrollbarAxisOffset!.dx, position.viewportDimension, _thumbPress!)) {
+          _updateScrollPosition(localPosition);
+        }
+        break;
+    }
     _dragScrollbarAxisOffset = localPosition;
   }
 
+  bool _canDragThumb(double dragOffset, double viewport, double thumbPress) {
+    return dragOffset >= thumbPress
+      && dragOffset <= viewport - (scrollbarPainter._thumbExtent - thumbPress);
+  }
+
   /// Handler called when a long press has ended.
   @protected
   @mustCallSuper
diff --git a/packages/flutter/test/widgets/scrollbar_test.dart b/packages/flutter/test/widgets/scrollbar_test.dart
index 2c32042..2d0c86d 100644
--- a/packages/flutter/test/widgets/scrollbar_test.dart
+++ b/packages/flutter/test/widgets/scrollbar_test.dart
@@ -2717,4 +2717,142 @@
 
     expect(scrollController.offset, 0.0);
   });
+
+  testWidgets('Scrollbar thumb can only be dragged from long press point', (WidgetTester tester) async {
+    // Regression test for https://github.com/flutter/flutter/issues/107765
+
+    final ScrollController scrollController = ScrollController();
+    final UniqueKey uniqueKey = UniqueKey();
+    await tester.pumpWidget(
+      Directionality(
+        textDirection: TextDirection.ltr,
+        child: MediaQuery(
+          data: const MediaQueryData(),
+          child: ScrollConfiguration(
+            behavior: const ScrollBehavior().copyWith(
+              scrollbars: false,
+            ),
+            child: PrimaryScrollController(
+              controller: scrollController,
+              child: RawScrollbar(
+                isAlwaysShown: true,
+                controller: scrollController,
+                child: CustomScrollView(
+                  primary: true,
+                  slivers: <Widget>[
+                    SliverToBoxAdapter(
+                      child: Container(
+                        height: 600.0,
+                      ),
+                    ),
+                    SliverToBoxAdapter(
+                      key: uniqueKey,
+                      child: Container(
+                        height: 600.0,
+                      ),
+                    ),
+                    SliverToBoxAdapter(
+                      child: Container(
+                        height: 600.0,
+                      ),
+                    ),
+                  ],
+                ),
+              ),
+            ),
+          ),
+        ),
+      ),
+    );
+    await tester.pumpAndSettle();
+    expect(scrollController.offset, 0.0);
+    expect(
+      find.byType(RawScrollbar),
+      paints
+        ..rect(rect: const Rect.fromLTRB(794.0, 0.0, 800.0, 600.0))
+        ..rect(
+          rect: const Rect.fromLTRB(794.0, 0.0, 800.0, 200.0),
+          color: const Color(0x66BCBCBC),
+        ),
+    );
+
+    // Long press on the thumb in the center and drag down to the bottom.
+    const double scrollAmount = 400.0;
+    final TestGesture dragScrollbarGesture = await tester.startGesture(const Offset(797.0, 100.0));
+    await tester.pumpAndSettle();
+    await dragScrollbarGesture.moveBy(const Offset(0.0, scrollAmount));
+    await tester.pumpAndSettle();
+
+    // Drag down past the long press point.
+    await dragScrollbarGesture.moveBy(const Offset(0.0, 100));
+    await tester.pumpAndSettle();
+
+    // Drag up without reaching press point on the thumb.
+    await dragScrollbarGesture.moveBy(const Offset(0.0, -50));
+    await tester.pumpAndSettle();
+
+    // Thumb should not move yet.
+    expect(
+      find.byType(RawScrollbar),
+      paints
+        ..rect(rect: const Rect.fromLTRB(794.0, 0.0, 800.0, 600.0))
+        ..rect(
+          rect: const Rect.fromLTRB(794.0, 400.0, 800.0, 600.0),
+          color: const Color(0x66BCBCBC),
+        ),
+    );
+
+    // Drag up to reach press point on the thumb.
+    await dragScrollbarGesture.moveBy(const Offset(0.0, -50));
+    await tester.pumpAndSettle();
+
+    // Drag up.
+    await dragScrollbarGesture.moveBy(const Offset(0.0, -300));
+    await tester.pumpAndSettle();
+
+    // Thumb should be moved.
+    expect(
+      find.byType(RawScrollbar),
+      paints
+        ..rect(rect: const Rect.fromLTRB(794.0, 0.0, 800.0, 600.0))
+        ..rect(
+          rect: const Rect.fromLTRB(794.0, 100.0, 800.0, 300.0),
+          color: const Color(0x66BCBCBC),
+        ),
+    );
+
+    // Drag up to reach the top and exceed the long press point.
+    await dragScrollbarGesture.moveBy(const Offset(0.0, -200));
+    await tester.pumpAndSettle();
+
+    // Drag down to reach the long press point.
+    await dragScrollbarGesture.moveBy(const Offset(0.0, 100));
+    await tester.pumpAndSettle();
+
+    // Thumb should not move yet.
+    expect(
+      find.byType(RawScrollbar),
+      paints
+        ..rect(rect: const Rect.fromLTRB(794.0, 0.0, 800.0, 600.0))
+        ..rect(
+          rect: const Rect.fromLTRB(794.0, 0.0, 800.0, 200.0),
+          color: const Color(0x66BCBCBC),
+        ),
+    );
+
+    // Drag down past the long press point.
+    await dragScrollbarGesture.moveBy(const Offset(0.0, 100));
+    await tester.pumpAndSettle();
+
+    // Thumb should be moved.
+    expect(
+      find.byType(RawScrollbar),
+      paints
+        ..rect(rect: const Rect.fromLTRB(794.0, 0.0, 800.0, 600.0))
+        ..rect(
+          rect: const Rect.fromLTRB(794.0, 100.0, 800.0, 300.0),
+          color: const Color(0x66BCBCBC),
+        ),
+    );
+  }, variant: TargetPlatformVariant.desktop());
 }