diff --git a/common/view/wx_view_controls.cpp b/common/view/wx_view_controls.cpp
index b4cc2d00bf..70ed240f0a 100644
--- a/common/view/wx_view_controls.cpp
+++ b/common/view/wx_view_controls.cpp
@@ -353,7 +353,7 @@ void WX_VIEW_CONTROLS::onWheel( wxMouseEvent& aEvent )
 
         // Refresh the zoom level and mouse position on message panel
         // (mouse position has not changed, only the zoom level has changed):
-        refreshMouse();
+        refreshMouse( true );
     }
     else
     {
@@ -371,7 +371,7 @@ void WX_VIEW_CONTROLS::onWheel( wxMouseEvent& aEvent )
         VECTOR2D delta( scrollX, scrollY );
 
         m_view->SetCenter( m_view->GetCenter() + delta );
-        refreshMouse();
+        refreshMouse( true );
     }
 
     // Do not skip this event, otherwise wxWidgets will fire
@@ -545,7 +545,7 @@ void WX_VIEW_CONTROLS::onTimer( wxTimerEvent& aEvent )
         dir = m_view->ToWorld( dir, false );
         m_view->SetCenter( m_view->GetCenter() + dir );
 
-        refreshMouse();
+        refreshMouse( true );
     }
     break;
 
@@ -766,7 +766,9 @@ void WX_VIEW_CONTROLS::WarpCursor( const VECTOR2D& aPosition, bool aWorldCoordin
         KIPLATFORM::UI::WarpPointer( m_parentPanel, aPosition.x, aPosition.y );
     }
 
-    refreshMouse();
+    // If we are not refreshing because of mouse movement, don't set the modifiers
+    // because we are refreshing for keyboard movement, which uses the same modifiers for other actions
+    refreshMouse( m_updateCursor );
 }
 
 
@@ -894,7 +896,7 @@ void WX_VIEW_CONTROLS::handleCursorCapture( int x, int y )
 }
 
 
-void WX_VIEW_CONTROLS::refreshMouse()
+void WX_VIEW_CONTROLS::refreshMouse( bool aSetModifiers )
 {
     // Notify tools that the cursor position has changed in the world coordinates
     wxMouseEvent moveEvent( EVT_REFRESH_MOUSE );
@@ -902,10 +904,13 @@ void WX_VIEW_CONTROLS::refreshMouse()
     moveEvent.SetX( msp.x );
     moveEvent.SetY( msp.y );
 
-    // Set the modifiers state
-    moveEvent.SetControlDown( wxGetKeyState( WXK_CONTROL ) );
-    moveEvent.SetShiftDown( wxGetKeyState( WXK_SHIFT ) );
-    moveEvent.SetAltDown( wxGetKeyState( WXK_ALT ) );
+    if( aSetModifiers )
+    {
+        // Set the modifiers state
+        moveEvent.SetControlDown( wxGetKeyState( WXK_CONTROL ) );
+        moveEvent.SetShiftDown( wxGetKeyState( WXK_SHIFT ) );
+        moveEvent.SetAltDown( wxGetKeyState( WXK_ALT ) );
+    }
 
     m_cursorPos = GetClampedCoords( m_view->ToWorld( VECTOR2D( msp.x, msp.y ) ) );
     wxPostEvent( m_parentPanel, moveEvent );
@@ -954,7 +959,7 @@ void WX_VIEW_CONTROLS::UpdateScrollbars()
         // Trigger a mouse refresh to get the canvas update in GTK (re-draws the scrollbars).
         // Note that this causes an infinite loop on OSX and Windows (in certain cases) as it
         // generates a paint event.
-        refreshMouse();
+        refreshMouse( false );
 #endif
     }
 }
diff --git a/include/view/wx_view_controls.h b/include/view/wx_view_controls.h
index 773258c425..e96e89aa23 100644
--- a/include/view/wx_view_controls.h
+++ b/include/view/wx_view_controls.h
@@ -151,8 +151,9 @@ private:
      * It is mostly used for notifying the tools that the cursor position in the world
      * coordinates has changed, whereas the screen coordinates remained the same (e.g.
      * frame edge autopanning).
+     * @param aSetModifiers If false, don't change the modifiers (they were set using the keyboard motion)
      */
-    void refreshMouse();
+    void refreshMouse( bool aSetModifiers);
 
     /**
      * Get the cursor position in the screen coordinates.