diff --git a/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp b/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp
index b7d4b7230247cd..1de1683444196f 100644
--- a/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp
+++ b/src/plugins/intel_gpu/src/graph/layout_optimizer.cpp
@@ -924,6 +924,10 @@ static bool is_node_for_onednn(fully_connected_node const& node) {
             auto decompression_zp_dt = node.get_input_layout(decompression_zp_idx).data_type;
             if (weights_dt != decompression_zp_dt)
                 return false;
+
+            auto input_dt = node.get_input_layout(0).data_type;
+            if (input_dt == data_types::f32)
+                return false;
         }
     }