diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
index 618838c48313..1729eded34ab 100644
--- a/include/linux/cgroup.h
+++ b/include/linux/cgroup.h
@@ -836,7 +836,11 @@ static inline struct cgroup *sock_cgroup_ptr(struct sock_cgroup_data *skcd)
 	 */
 	v = READ_ONCE(skcd->val);
 
-	if (v & 3)
+#ifdef __LITTLE_ENDIAN
+	if (v & 0x3)
+#else
+	if (v & 0xc0)
+#endif
 		return &cgrp_dfl_root.cgrp;
 
 	return (struct cgroup *)(unsigned long)v ?: &cgrp_dfl_root.cgrp;