diff --git a/src/useMap.ts b/src/useMap.ts
index ded74ed239..b4419793f8 100644
--- a/src/useMap.ts
+++ b/src/useMap.ts
@@ -12,34 +12,35 @@ export interface Actions<T extends object> extends StableActions<T> {
 }
 
 const useMap = <T extends object = any>(initialMap: T = {} as T): [T, Actions<T>] => {
-  const [map, set] = useState<T>(initialMap);
+  const [map, setMap] = useState<T>(() => ({ ...initialMap }));
 
   const stableActions = useMemo<StableActions<T>>(
     () => ({
-      set: (key, entry) => {
-        set((prevMap) => ({
-          ...prevMap,
-          [key]: entry,
-        }));
-      },
-      setAll: (newMap: T) => {
-        set(newMap);
-      },
-      remove: (key) => {
-        set((prevMap) => {
-          const { [key]: omit, ...rest } = prevMap;
+      set: <K extends keyof T>(key: K, value: T[K]) =>
+        setMap((prev) => {
+          // Use Object.is for correct NaN handling
+          if (Object.is(prev[key], value)) return prev;
+          return {
+            ...prev,
+            [key]: value,
+          };
+        }),
+      setAll: (newMap: T) => setMap(newMap),
+      remove: <K extends keyof T>(key: K) =>
+        setMap((prev) => {
+          if (!(key in prev)) return prev;
+          const { [key]: omit, ...rest } = prev;
           return rest as T;
-        });
-      },
-      reset: () => set(initialMap),
+        }),
+      reset: () => setMap({ ...initialMap }),
     }),
-    [set]
+    [initialMap]
   );
 
-  const utils = {
-    get: useCallback((key) => map[key], [map]),
-    ...stableActions,
-  } as Actions<T>;
+  const get = useCallback(<K extends keyof T>(key: K): T[K] => map[key], [map]);
+
+  // Memoize the entire utils object to maintain stable reference
+  const utils = useMemo<Actions<T>>(() => ({ get, ...stableActions }), [get, stableActions]);
 
   return [map, utils];
 };
diff --git a/src/useSet.ts b/src/useSet.ts
index 9c88306cc9..47ef8e3cd1 100644
--- a/src/useSet.ts
+++ b/src/useSet.ts
@@ -13,26 +13,40 @@ export interface Actions<K> extends StableActions<K> {
 }
 
 const useSet = <K>(initialSet = new Set<K>()): [Set<K>, Actions<K>] => {
-  const [set, setSet] = useState(initialSet);
-
-  const stableActions = useMemo<StableActions<K>>(() => {
-    const add = (item: K) => setSet((prevSet) => new Set([...Array.from(prevSet), item]));
-    const remove = (item: K) =>
-      setSet((prevSet) => new Set(Array.from(prevSet).filter((i) => i !== item)));
-    const toggle = (item: K) =>
-      setSet((prevSet) =>
-        prevSet.has(item)
-          ? new Set(Array.from(prevSet).filter((i) => i !== item))
-          : new Set([...Array.from(prevSet), item])
-      );
-
-    return { add, remove, toggle, reset: () => setSet(initialSet), clear: () => setSet(new Set()) };
-  }, [setSet]);
-
-  const utils = {
-    has: useCallback((item) => set.has(item), [set]),
-    ...stableActions,
-  } as Actions<K>;
+  const [set, setSet] = useState(() => new Set(initialSet));
+
+  const stableActions = useMemo<StableActions<K>>(
+    () => ({
+      add: (item: K) =>
+        setSet((prev) => {
+          if (prev.has(item)) return prev;
+          const next = new Set(prev);
+          next.add(item);
+          return next;
+        }),
+      remove: (item: K) =>
+        setSet((prev) => {
+          if (!prev.has(item)) return prev;
+          const next = new Set(prev);
+          next.delete(item);
+          return next;
+        }),
+      toggle: (item: K) =>
+        setSet((prev) => {
+          const next = new Set(prev);
+          prev.has(item) ? next.delete(item) : next.add(item);
+          return next;
+        }),
+      reset: () => setSet(new Set(initialSet)),
+      clear: () => setSet((prev) => (prev.size === 0 ? prev : new Set())),
+    }),
+    [initialSet]
+  );
+
+  const has = useCallback((item: K) => set.has(item), [set]);
+
+  // Memoize the entire utils object to maintain stable reference
+  const utils = useMemo<Actions<K>>(() => ({ has, ...stableActions }), [has, stableActions]);
 
   return [set, utils];
 };