diff --git a/src/components/Actions.tsx b/src/components/Actions.tsx
index fcdb98ef6..b29207207 100644
--- a/src/components/Actions.tsx
+++ b/src/components/Actions.tsx
@@ -217,12 +217,13 @@ export const ShapesSwitcher = ({
             aria-keyshortcuts={shortcut}
             data-testid={value}
             onChange={({ pointerType }) => {
+              if (!penDetected && pointerType === "pen") {
+                setAppState({ penMode: true });
+              }
               setAppState({
                 elementType: value,
                 multiElement: null,
                 selectedElementIds: {},
-                penMode:
-                  (!penDetected && pointerType === "pen") ?? appState.penMode,
               });
               setCursorForShape(canvas, { ...appState, elementType: value });
               if (value === "image") {