diff --git a/hoverxref/domains.py b/hoverxref/domains.py
index c70af5c7..f8c36515 100644
--- a/hoverxref/domains.py
+++ b/hoverxref/domains.py
@@ -52,6 +52,16 @@ def _get_docpath(self, builder, docname):
         return docpath
 
     def _is_ignored_ref(self, env, target):
+        # HACK: skip all references if the builder is non-html. We shouldn't
+        # have overridden the Domain in first instance at ``setup_domains``
+        # function, but at that time ``app.builder`` is not yet initialized. If
+        # we suscribe ourselves to ``builder-initied`` it's too late and our
+        # override does not take effect. Other builders (e.g. LatexBuilder) may
+        # fail with internal functions we use (e.g. builder.get_outfilename).
+        # So, we are skipping it here :(
+        if env.app.builder.format != 'html':
+            return True
+
         if target in env.config.hoverxref_ignore_refs:
             logger.info(
                 'Ignoring reference in hoverxref_ignore_refs. target=%s',
diff --git a/tests/examples/default/index.rst b/tests/examples/default/index.rst
index d528f10a..4461a286 100644
--- a/tests/examples/default/index.rst
+++ b/tests/examples/default/index.rst
@@ -9,3 +9,10 @@ Using ``hoverxref`` (or ``ref`` if ``hoverxref_auto_ref=True``) should add an ``
 :ref:`This a :ref: to Chapter I <Chapter I>`.
 
 :hoverxref:`This a :hoverxref: to Chapter I, Section I <Section I>`.
+
+.. _example-reference:
+
+Example Reference
+-----------------
+
+This is a reference to :ref:`example-reference`.
diff --git a/tests/test_internals.py b/tests/test_internals.py
index 5df069e5..d031a88e 100644
--- a/tests/test_internals.py
+++ b/tests/test_internals.py
@@ -2,6 +2,7 @@
 import os
 import pytest
 import shutil
+from unittest import mock
 
 from hoverxref.translators import HoverXRefHTMLTranslatorMixin
 
@@ -44,3 +45,36 @@ def test_override_translator_non_html_builder(app, status, warning):
     assert app.builder.format == 'html'
     for name, klass in app.registry.translators.items():
         assert issubclass(klass, HoverXRefHTMLTranslatorMixin)
+
+
+@pytest.mark.sphinx(
+    srcdir=srcdir,
+    buildername='latex',
+    confoverrides={
+        'hoverxref_project': 'myproject',
+        'hoverxref_version': 'myversion',
+        'hoverxref_auto_ref': True,
+    },
+)
+def test_dont_fail_non_html_builder(app, status, warning):
+    """
+    Test our resolver is not used by non-HTML builder.
+
+    When running the build with ``latex`` as builder and
+    ``hoverxref_auto_ref=True`` it should not fail with
+
+    def _get_docpath(self, builder, docname):
+        docpath = builder.get_outfilename(docname)
+        AttributeError: 'LaTeXBuilder' object has no attribute 'get_outfilename'
+
+    LaTeXBuilder should never use our resolver.
+    """
+
+    with mock.patch('hoverxref.domains.HoverXRefBaseDomain._get_docpath') as _get_docpath:
+        app.build()
+        assert not _get_docpath.called
+    path = app.outdir / 'test.tex'
+    assert path.exists() is True
+    content = open(path).read()
+
+    assert app.builder.format == 'latex'