diff --git a/app/assets/stylesheets/settings.css b/app/assets/stylesheets/settings.css
index 983034a03..f2f1be59e 100644
--- a/app/assets/stylesheets/settings.css
+++ b/app/assets/stylesheets/settings.css
@@ -67,10 +67,6 @@
       /* Add padding if it's not already on a link within */
       &:not(:has(a:first-child)) { padding-inline-end: var(--settings-item-padding-inline); }
       &:not(:has(a:last-child)) { padding-inline-end: var(--settings-item-padding-inline); }
-
-      &[aria-selected] {
-        background: var(--color-selected);
-      }
     }
 
     a {
@@ -82,5 +78,12 @@
         }
       }
     }
+
+    /* Only add a BG color when you can actually navigate */
+    .settings__user-filter:focus-within & {
+      [aria-selected] {
+        background: var(--color-selected);
+      }
+    }
   }
 }