fix ATA toplevel dep detection

ATA tried to use the `_requiredBy` field to determine toplevel deps,
but this is not portable. Not only is it unavailable in npm@>=7, but
neither Yarn nor pnpm write this metadata to node_modules pkgjsons.

Fixes: microsoft#44130

Author: zkat

src/jsTyping/jsTyping.ts

@@ -9,7 +9,6 @@ namespace ts.JsTyping {
     }
 
     interface PackageJson {
-        _requiredBy?: string[];
         dependencies?: MapLike<string>;
         devDependencies?: MapLike<string>;
         name?: string;
@@ -152,17 +151,8 @@ namespace ts.JsTyping {
         const possibleSearchDirs = new Set(fileNames.map(getDirectoryPath));
         possibleSearchDirs.add(projectRootPath);
         possibleSearchDirs.forEach((searchDir) => {
-            const packageJsonPath = combinePaths(searchDir, "package.json");
-            getTypingNamesFromJson(packageJsonPath, filesToWatch);
-
-            const bowerJsonPath = combinePaths(searchDir, "bower.json");
-            getTypingNamesFromJson(bowerJsonPath, filesToWatch);
-
-            const bowerComponentsPath = combinePaths(searchDir, "bower_components");
-            getTypingNamesFromPackagesFolder(bowerComponentsPath, filesToWatch);
-
-            const nodeModulesPath = combinePaths(searchDir, "node_modules");
-            getTypingNamesFromPackagesFolder(nodeModulesPath, filesToWatch);
+            getTypingNames(searchDir, "package.json", "node_modules", filesToWatch);
+            getTypingNames(searchDir, "bower.json", "bower_components", filesToWatch);
         });
         if(!typeAcquisition.disableFilenameBasedTypeAcquisition) {
             getTypingNamesFromSourceFileNames(fileNames);
@@ -214,17 +204,103 @@ namespace ts.JsTyping {
         }
 
         /**
-         * Get the typing info from common package manager json files like package.json or bower.json
+         * Adds inferred typings from manifest/module pairs (think package.json + node_modules)
+         *
+         * @param projectRootPath is the path to the directory where to look for package.json, bower.json and other typing information
+         * @param manifestName is the name of the manifest (package.json or bower.json)
+         * @param modulesDirName is the directory name for modules (node_modules or bower_components)
+         * @param filesToWatch are the files to watch for changes. We will push things into this array.
          */
-        function getTypingNamesFromJson(jsonPath: string, filesToWatch: Push<string>) {
-            if (!host.fileExists(jsonPath)) {
+        function getTypingNames(projectRootPath: string, manifestName: string, modulesDirName: string, filesToWatch: string[]): void {
+
+            // First, we check the manifests themselves. They're not
+            // _required_, but they allow us to do some filtering when dealing
+            // with big flat dep directories.
+            const jsonPath = combinePaths(projectRootPath, manifestName);
+            let jsonConfig;
+            let jsonTypingNames;
+            if (host.fileExists(jsonPath)) {
+                filesToWatch.push(jsonPath);
+                jsonConfig = readConfigFile(jsonPath, path => host.readFile(path)).config;
+                jsonTypingNames = flatMap([jsonConfig.dependencies, jsonConfig.devDependencies, jsonConfig.optionalDependencies, jsonConfig.peerDependencies], getOwnKeys);
+                addInferredTypings(jsonTypingNames, `Typing names in '${jsonPath}' dependencies`);
+            }
+            else {
+                jsonConfig = undefined;
+            }
+
+
+            // Now we scan the directories for typing information in
+            // already-installed dependencies (if present). Note that this
+            // step happens regardless of whether a manifest was present,
+            // which is certainly a valid configuration, if an unusual one.
+            const packagesFolderPath = combinePaths(projectRootPath, modulesDirName);
+            if (!host.directoryExists(packagesFolderPath)) {
                 return;
             }
+            if (log) log(`Searching for typing names in ${packagesFolderPath}`);
+
+            // There's two cases we have to take into account here:
+            // 1. If jsonConfig is undefined, then we're not using a manifest.
+            //    That means that we should scan _all_ dependencies at the top
+            //    level of the modulesDir.
+            // 2. If jsonConfig is defined, then we can do some special
+            //    filtering to reduce the amount of scanning we need to do.
+            //
+            // Previous versions of this algorithm checked for a `_requiredBy`
+            // field in the package.json, but that field is only present in
+            // `npm@>=3 <7`.
+
+            // Package names that do **not** provide their own typings, so
+            // we'll look them up.
+            const packageNames: string[] = [];
 
-            filesToWatch.push(jsonPath);
-            const jsonConfig: PackageJson = readConfigFile(jsonPath, path => host.readFile(path)).config;
-            const jsonTypingNames = flatMap([jsonConfig.dependencies, jsonConfig.devDependencies, jsonConfig.optionalDependencies, jsonConfig.peerDependencies], getOwnKeys);
-            addInferredTypings(jsonTypingNames, `Typing names in '${jsonPath}' dependencies`);
+            const dependencyManifestNames = jsonTypingNames
+                // This is #1 described above.
+                ? jsonTypingNames.map(typingName => combinePaths(packagesFolderPath, typingName, manifestName))
+                // And #2. Depth = 3 because scoped packages look like `node_modules/@foo/bar/package.json`
+                : host.readDirectory(packagesFolderPath, [Extension.Json], /*excludes*/ undefined, /*includes*/ undefined, /*depth*/ 3)
+                    .filter(jsonPath => {
+                        // It's ok to treat
+                        // `node_modules/@foo/bar/package.json` as a manifest,
+                        // but not `node_modules/jquery/nested/package.json`.
+                        // We only assume depth 3 is ok for formally scoped
+                        // packages. So that needs this dance here.
+                        const pathComponents = getPathComponents(normalizePath(jsonPath));
+                        const isScoped = pathComponents[1][0] === "@";
+                        if (
+                            (isScoped && pathComponents.length === 4)
+                            || (!isScoped && pathComponents.length === 3)
+                        ) {
+                            return getBaseFileName(jsonPath) === manifestName;
+                        }
+                        else {
+                            return false;
+                        }
+                    });
+
+            for (const manifestPath of dependencyManifestNames) {
+                const normalizedFileName = normalizePath(manifestPath);
+                const result = readConfigFile(normalizedFileName, (path: string) => host.readFile(path));
+                const manifest: PackageJson = result.config;
+
+                // If the package has its own d.ts typings, those will take precedence. Otherwise the package name will be used
+                // to download d.ts files from DefinitelyTyped
+                if (!manifest.name) {
+                    continue;
+                }
+                const ownTypes = manifest.types || manifest.typings;
+                if (ownTypes) {
+                    const absolutePath = getNormalizedAbsolutePath(ownTypes, getDirectoryPath(normalizedFileName));
+                    if (log) log(`    Package '${manifest.name}' provides its own types.`);
+                    inferredTypings.set(manifest.name, absolutePath);
+                }
+                else {
+                    packageNames.push(manifest.name);
+                }
+            }
+
+            addInferredTypings(packageNames, "    Found package names");
         }
 
         /**
@@ -251,58 +327,6 @@ namespace ts.JsTyping {
                 addInferredTyping("react");
             }
         }
-
-        /**
-         * Infer typing names from packages folder (ex: node_module, bower_components)
-         * @param packagesFolderPath is the path to the packages folder
-         */
-        function getTypingNamesFromPackagesFolder(packagesFolderPath: string, filesToWatch: Push<string>) {
-            filesToWatch.push(packagesFolderPath);
-
-            // Todo: add support for ModuleResolutionHost too
-            if (!host.directoryExists(packagesFolderPath)) {
-                return;
-            }
-
-            // depth of 2, so we access `node_modules/foo` but not `node_modules/foo/bar`
-            const fileNames = host.readDirectory(packagesFolderPath, [Extension.Json], /*excludes*/ undefined, /*includes*/ undefined, /*depth*/ 2);
-            if (log) log(`Searching for typing names in ${packagesFolderPath}; all files: ${JSON.stringify(fileNames)}`);
-            const packageNames: string[] = [];
-            for (const fileName of fileNames) {
-                const normalizedFileName = normalizePath(fileName);
-                const baseFileName = getBaseFileName(normalizedFileName);
-                if (baseFileName !== "package.json" && baseFileName !== "bower.json") {
-                    continue;
-                }
-                const result = readConfigFile(normalizedFileName, (path: string) => host.readFile(path));
-                const packageJson: PackageJson = result.config;
-
-                // npm 3's package.json contains a "_requiredBy" field
-                // we should include all the top level module names for npm 2, and only module names whose
-                // "_requiredBy" field starts with "#" or equals "/" for npm 3.
-                if (baseFileName === "package.json" && packageJson._requiredBy &&
-                    filter(packageJson._requiredBy, (r: string) => r[0] === "#" || r === "/").length === 0) {
-                    continue;
-                }
-
-                // If the package has its own d.ts typings, those will take precedence. Otherwise the package name will be used
-                // to download d.ts files from DefinitelyTyped
-                if (!packageJson.name) {
-                    continue;
-                }
-                const ownTypes = packageJson.types || packageJson.typings;
-                if (ownTypes) {
-                    const absolutePath = getNormalizedAbsolutePath(ownTypes, getDirectoryPath(normalizedFileName));
-                    if (log) log(`    Package '${packageJson.name}' provides its own types.`);
-                    inferredTypings.set(packageJson.name, absolutePath);
-                }
-                else {
-                    packageNames.push(packageJson.name);
-                }
-            }
-            addInferredTypings(packageNames, "    Found package names");
-        }
-
     }
 
     export const enum NameValidationResult {

src/testRunner/unittests/tsserver/typingsInstaller.ts

@@ -839,10 +839,29 @@ namespace ts.projectSystem {
                 path: "/app.js",
                 content: ""
             };
+            const pkgJson = {
+                path: "/package.json",
+                content: JSON.stringify({
+                    dependencies: {
+                        jquery: "1.0.0"
+                    }
+                })
+            };
             const jsconfig = {
                 path: "/jsconfig.json",
                 content: JSON.stringify({})
             };
+            // Should only accept direct dependencies.
+            const commander = {
+                path: "/node_modules/commander/index.js",
+                content: ""
+            };
+            const commanderPackage = {
+                path: "/node_modules/commander/package.json",
+                content: JSON.stringify({
+                    name: "commander",
+                })
+            };
             const jquery = {
                 path: "/node_modules/jquery/index.js",
                 content: ""
@@ -860,10 +879,10 @@ namespace ts.projectSystem {
                 path: "/tmp/node_modules/@types/jquery/index.d.ts",
                 content: ""
             };
-            const host = createServerHost([app, jsconfig, jquery, jqueryPackage, nestedPackage]);
+            const host = createServerHost([app, jsconfig, pkgJson, commander, commanderPackage, jquery, jqueryPackage, nestedPackage]);
             const installer = new (class extends Installer {
                 constructor() {
-                    super(host, { globalTypingsCacheLocation: "/tmp", typesRegistry: createTypesRegistry("jquery", "nested") });
+                    super(host, { globalTypingsCacheLocation: "/tmp", typesRegistry: createTypesRegistry("jquery", "nested", "commander") });
                 }
                 installWorker(_requestId: number, args: string[], _cwd: string, cb: TI.RequestCompletedAction) {
                     assert.deepEqual(args, [`@types/jquery@ts${versionMajorMinor}`]);