diff --git a/setup.py b/setup.py
index f6adb4f86c3..ee7514080a8 100644
--- a/setup.py
+++ b/setup.py
@@ -225,7 +225,7 @@ def src_path(self, installer: "InstallerBuildExt") -> Path:
             self.src = self.src.replace("%BUILD_TYPE%", cfg)
         else:
             # Remove %BUILD_TYPE% from the path.
-            self.src = self.src.replace("/%BUILD_TYPE%", "")
+            self.src = self.src.replace("%BUILD_TYPE%/", "")
 
         # Construct the full source path, resolving globs. If there are no glob
         # pattern characters, this will just ensure that the source file exists.
@@ -263,7 +263,7 @@ def __init__(
                 output is in a subdirectory named after the build type. For single-
                 config generators (like Makefile Generators or Ninja), this placeholder
                 will be removed.
-            src_name: The name of the file to install
+            src_name: The name of the file to install.
             dst: The path to install to, relative to the root of the pip
                 package. If dst ends in "/", it is treated as a directory.
                 Otherwise it is treated as a filename.
@@ -305,13 +305,17 @@ def dst_path(self, installer: "InstallerBuildExt") -> Path:
 class BuiltExtension(_BaseExtension):
     """An extension that installs a python extension that was built by cmake."""
 
-    def __init__(self, src: str, modpath: str):
+    def __init__(self, src_dir: str, src_name: str, modpath: str):
         """Initializes a BuiltExtension.
 
         Args:
-            src: The path to the file to install (typically a shared library),
-                relative to the cmake-out directory. May be an fnmatch-style
-                glob that matches exactly one file. If the path ends in `.so`,
+            src_dir: The directory of the file to install, relative to the cmake-out
+                directory. A placeholder %BUILD_TYPE% will be replaced with the build
+                type for multi-config generators (like Visual Studio) where the build
+                output is in a subdirectory named after the build type. For single-
+                config generators (like Makefile Generators or Ninja), this placeholder
+                will be removed.
+            src_name: The name of the file to install. If the path ends in `.so`,
                 this class will also look for similarly-named `.dylib` files.
             modpath: The dotted path of the python module that maps to the
                 extension.
@@ -319,6 +323,7 @@ def __init__(self, src: str, modpath: str):
         assert (
             "/" not in modpath
         ), f"modpath must be a dotted python module path: saw '{modpath}'"
+        src = os.path.join(src_dir, src_name)
         # This is a real extension, so use the modpath as the name.
         super().__init__(src=src, dst=modpath, name=modpath)
 
@@ -634,7 +639,9 @@ def get_ext_modules() -> List[Extension]:
             # portable kernels, and a selection of backends. This lets users
             # load and execute .pte files from python.
             BuiltExtension(
-                "_portable_lib.*", "executorch.extension.pybindings._portable_lib"
+                src_dir="%BUILD_TYPE%/",
+                src_name="_portable_lib.cp*",
+                modpath="executorch.extension.pybindings._portable_lib",
             )
         )
     if ShouldBuild.llama_custom_ops():