diff --git a/replaygain/CMakeLists.txt b/replaygain/CMakeLists.txt
index fe8f0b996..4611764f4 100644
--- a/replaygain/CMakeLists.txt
+++ b/replaygain/CMakeLists.txt
@@ -79,9 +79,6 @@ if (FFMPEG_FOUND OR MPG123_FOUND)
         endif (WIN32)
     endif (ENABLE_KDE_SUPPORT)
 
-    if (NOT ENABLE_KDE_SUPPORT AND NOT WIN32)
-        target_link_libraries(cantata-replaygain X11)
-    endif (NOT ENABLE_KDE_SUPPORT AND NOT WIN32)
     if (FFMPEG_FOUND)
         target_link_libraries(cantata-replaygain ${FFMPEG_LIBRARIES})
     endif (FFMPEG_FOUND)