diff --git a/src/components/App.tsx b/src/components/App.tsx
index 1fab2a2cf..700d73a93 100644
--- a/src/components/App.tsx
+++ b/src/components/App.tsx
@@ -226,6 +226,7 @@ import {
   setEraserCursor,
   updateActiveTool,
   getShortcutKey,
+  isTransparent,
 } from "../utils";
 import {
   ContextMenu,
@@ -264,6 +265,7 @@ import {
   getContainerCenter,
   getContainerDims,
   getTextBindableContainerAtPosition,
+  isHittingContainerStroke,
   isValidTextContainer,
 } from "../element/textElement";
 import { isHittingElementNotConsideringBoundingBox } from "../element/collision";
@@ -2762,7 +2764,19 @@ class App extends React.Component<AppProps, AppState> {
         sceneY,
       );
       if (container) {
-        if (isArrowElement(container) || hasBoundTextElement(container)) {
+        if (
+          isArrowElement(container) ||
+          hasBoundTextElement(container) ||
+          !isTransparent(
+            (container as ExcalidrawTextContainer).backgroundColor,
+          ) ||
+          isHittingContainerStroke(
+            sceneX,
+            sceneY,
+            container,
+            this.state.zoom.value,
+          )
+        ) {
           const midPoint = getContainerCenter(container, this.state);
 
           sceneX = midPoint.x;
diff --git a/src/element/textElement.ts b/src/element/textElement.ts
index c726c1c3f..3b591ff9b 100644
--- a/src/element/textElement.ts
+++ b/src/element/textElement.ts
@@ -11,7 +11,7 @@ import { mutateElement } from "./mutateElement";
 import { BOUND_TEXT_PADDING, TEXT_ALIGN, VERTICAL_ALIGN } from "../constants";
 import { MaybeTransformHandleType } from "./transformHandles";
 import Scene from "../scene/Scene";
-import { isTextElement } from ".";
+import { getElementBounds, isTextElement } from ".";
 import { getMaxContainerHeight, getMaxContainerWidth } from "./newElement";
 import {
   isBoundToContainer,
@@ -723,3 +723,62 @@ export const isValidTextContainer = (element: ExcalidrawElement) => {
     isArrowElement(element)
   );
 };
+
+export const isHittingContainerStroke = (
+  x: number,
+  y: number,
+  container: ExcalidrawTextContainer,
+  zoom: number,
+) => {
+  const threshold = 10 / zoom;
+  const bounds = getElementBounds(container);
+  const topLeft = [bounds[0], bounds[1]];
+  const topRight = [bounds[2], bounds[1]];
+  const bottomLeft = [bounds[0], bounds[3]];
+  const bottomRight = [bounds[2], bounds[3]];
+
+  const strokeWidth = container.strokeWidth;
+  if (container.type === "ellipse") {
+    return false;
+  }
+
+  // Left Stroke
+  if (
+    x >= topLeft[0] - threshold &&
+    x <= topLeft[0] + strokeWidth + threshold &&
+    y >= topLeft[1] - threshold &&
+    y <= bottomRight[1] + threshold
+  ) {
+    return true;
+  }
+  // Top stroke
+  if (
+    x >= topLeft[0] - threshold &&
+    x <= topRight[0] + threshold &&
+    y >= topLeft[1] - threshold &&
+    y <= topLeft[1] + threshold + strokeWidth
+  ) {
+    return true;
+  }
+
+  // Right stroke
+  if (
+    x >= topRight[0] - threshold - strokeWidth &&
+    x <= topRight[0] + threshold &&
+    y >= topRight[1] - threshold &&
+    y <= bottomRight[1] + threshold
+  ) {
+    return true;
+  }
+
+  // Bottom Stroke
+  if (
+    x >= bottomLeft[0] - threshold &&
+    x <= bottomRight[0] + threshold &&
+    y >= bottomLeft[1] - threshold - strokeWidth &&
+    y <= bottomLeft[1] + threshold
+  ) {
+    return true;
+  }
+  return false;
+};