diff --git a/src/modules.rs b/src/modules.rs
index 0c35cc2d073..f22435a10db 100644
--- a/src/modules.rs
+++ b/src/modules.rs
@@ -264,12 +264,19 @@ impl<'ast, 'sess, 'c> ModResolver<'ast, 'sess> {
                 path,
                 directory_ownership,
                 ..
-            }) => Ok(if mods_outside_ast.is_empty() {
-                SubModKind::External(path, directory_ownership)
-            } else {
-                mods_outside_ast.push((path, directory_ownership, sub_mod.clone()));
-                SubModKind::MultiExternal(mods_outside_ast)
-            }),
+            }) => {
+                if mods_outside_ast.is_empty() {
+                    Ok(SubModKind::External(path, directory_ownership))
+                } else {
+                    let should_insert = !mods_outside_ast
+                        .iter()
+                        .any(|(outside_path, _, _)| outside_path == &path);
+                    if should_insert {
+                        mods_outside_ast.push((path, directory_ownership, sub_mod.clone()));
+                    }
+                    Ok(SubModKind::MultiExternal(mods_outside_ast))
+                }
+            }
             Err(_) if !mods_outside_ast.is_empty() => {
                 Ok(SubModKind::MultiExternal(mods_outside_ast))
             }
diff --git a/tests/config/issue-3956.toml b/tests/config/issue-3956.toml
new file mode 100644
index 00000000000..7669890586b
--- /dev/null
+++ b/tests/config/issue-3956.toml
@@ -0,0 +1,5 @@
+ignore = [
+  "tests/**/issue-3956/graphics.rs",
+  "tests/**/issue-3956/graphics_emu.rs"
+]
+recursive = true
diff --git a/tests/target/issue-3956/graphics.rs b/tests/target/issue-3956/graphics.rs
new file mode 100644
index 00000000000..ea98ccf2349
--- /dev/null
+++ b/tests/target/issue-3956/graphics.rs
@@ -0,0 +1,2 @@
+// rustfmt-config: issue-3956.toml
+pub struct A;
diff --git a/tests/target/issue-3956/graphics_emu.rs b/tests/target/issue-3956/graphics_emu.rs
new file mode 100644
index 00000000000..88e41b515dc
--- /dev/null
+++ b/tests/target/issue-3956/graphics_emu.rs
@@ -0,0 +1,2 @@
+// rustfmt-config: issue-3956.toml
+pub struct B;
diff --git a/tests/target/issue-3956/lib.rs b/tests/target/issue-3956/lib.rs
new file mode 100644
index 00000000000..b0a26c9f0ba
--- /dev/null
+++ b/tests/target/issue-3956/lib.rs
@@ -0,0 +1,4 @@
+// rustfmt-config: issue-3956.toml
+#[cfg_attr(windows, path = "graphics.rs")]
+#[cfg_attr(not(windows), path = "graphics_emu.rs")]
+mod graphics;