diff --git a/.config/zsh/.zprofile b/.config/zsh/.zprofile
index e2d40bb..750ea9f 100644
--- a/.config/zsh/.zprofile
+++ b/.config/zsh/.zprofile
@@ -83,7 +83,8 @@ export PASSWORD_STORE_GENERATED_LENGTH="43"
 export LESS="--tabs=4 --RAW-CONTROL-CHARS --LONG-PROMPT"
 
 if command -v highlight > /dev/null; then
-    LESSOPEN="| $(command -v highlight) --force --out-format=truecolor --style=base16/unikitty-reversible --stdout %s"
+    # ~/.lessfilter handles the highlighting
+    LESSOPEN="| $(command -v lesspipe) %s"
 elif command -v pygmentize > /dev/null; then
     LESSOPEN="| $(command -v pygmentize) -f terminal16m -g -O style=paraiso-dark %s"
 elif command -v source-highlight > /dev/null; then
diff --git a/.lessfilter b/.lessfilter
new file mode 100755
index 0000000..ff7dffe
--- /dev/null
+++ b/.lessfilter
@@ -0,0 +1,20 @@
+#!/bin/zsh
+# Try to use highlight. If that fails or the output looks suspicious, use
+# lesspipe normally.
+
+command -v highlight > /dev/null || return 1
+file="${1}"
+
+local output="$(highlight --validate-input \
+                          --force \
+                          --out-format=truecolor \
+                          --style=base16/unikitty-reversible \
+                          --stdout "${file}")"
+[[ ${?} -eq 0 ]] || return 1
+
+# Try to catch when the output is just the magic string
+if [[ ${#output} -lt 256 && $(stat --printf="%s" "${file}") -gt 256 ]]; then
+    return 1
+fi
+
+print "${output}"