Add localization (#18954)

* Enable localization using nls

* Add i18n files

* Add current translations

* Add export and import scripts

* Fix import translation function

* Clean code

* Update package-lock.json

* Delete i18n files

* Update package-lock.json

* Add news

* Fix package files

* Addd sourcemaps

* Add missing translations

* Remove localize test

* Add paths in pre-release

Author: paulacamargo25

build/azure-pipeline.pre-release.yml

@@ -21,6 +21,28 @@ resources:
 extends:
   template: azure-pipelines/extension/pre-release.yml@templates
   parameters:
+    locPushSteps:
+      - script: echo '$(SetTranslationsPath.EnglishStrings)'
+      - displayName: Install NPM dependencies
+        script: 'cd $(EXTENSION_DIR) && npm ci'
+      - script: npm run translations-export
+        displayName: Run translations-export
+      - publish: $(VSCODE_TRANSLATIONS_EXPORT_DIR)
+        artifact: translations-export
+        displayName: Publish translations-export
+      - pwsh: |
+          $Destination = "$(SetTranslationsPath.EnglishStrings)"
+          New-Item -ItemType Directory -Force "$(SetTranslationsPath.EnglishStrings)"
+          Move-Item "$(Pipeline.Workspace)/ms-python.python" $Destination
+        displayName: Move vscode-python-translations-export to translations path
+
+    locPullSteps:
+        - script: echo '$(SetTranslationsPath.TranslatedStrings)'
+        - script: 'gulp translations-import --option "${SetTranslationsPath.TranslatedStrings}"'
+        - pwsh: ls
+          displayName: test if there is stuff
+          workingDirectory: '$(SetTranslationsPath.TranslatedStrings)'
+
     buildSteps:
       - task: NodeTool@0
         inputs:

build/loc/ExportTranslations.yml

@@ -0,0 +1,72 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+
+pr: none
+trigger:
+  branches:
+    include:
+      - main
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily midnight build
+  branches:
+    include:
+    - main
+
+stages:
+- stage: Localization
+  pool:
+    vmImage: windows-latest
+  variables:
+    skipComponentGovernanceDetection: true
+    VSCODE_EXTENSIONS_LOC_DIR: $(Pipeline.Workspace)/vscode-extensions-loc
+    EXTENSION_DIR: $(Build.SourcesDirectory)
+    VSCODE_TRANSLATIONS_EXPORT_DIR: $(Pipeline.Workspace)/ms-python.python
+  jobs:
+  - job: Localization
+    timeoutInMinutes: 180
+    steps:
+
+    # Clone extension Repository and run translations-export to get xlf file
+    - script: git clone https://github.com/microsoft/vscode-extensions-loc/ ${VSCODE_EXTENSIONS_LOC_DIR}
+      displayName: Clone Microsoft/vscode-extensions-loc
+
+
+    - task: NodeTool@0
+      inputs:
+        versionSpec: '14.18.2'
+      displayName: Select Node version
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: '3.7'
+        addToPath: true
+        architecture: 'x64'
+      displayName: Select Python version
+    - displayName: Install NPM dependencies
+      script: 'cd $(EXTENSION_DIR) && npm ci'
+    - script: npm run translations-export
+      displayName: Run translations-export
+    - publish: $(VSCODE_TRANSLATIONS_EXPORT_DIR)
+      artifact: translations-export
+      displayName: Publish translations-export
+
+    - pwsh: |
+        $Destination = "$(VSCODE_EXTENSIONS_LOC_DIR)/src/en"
+        New-Item -ItemType Directory -Force "$(VSCODE_EXTENSIONS_LOC_DIR)/src/en"
+        Move-Item "$(VSCODE_TRANSLATIONS_EXPORT_DIR)" $Destination
+      displayName: Move vscode-python-translations-export to expected path
+
+    - name: Create Pull Request on vscode-extensions-loc
+        id: cpr
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{ secrets.REPO_PAT }}
+          path: ${{ github.workspace }}/vscode-extensions-loc
+          title: Update Localization xlf files
+          body: Automated changes based on Microsoft/vscode-python@${{ github.sha }}
+          branch: update-localization-files-microsoft-vscode-python-${{ github.sha }}
+          commit-message: update localization xlf files Microsoft/vscode-python@${{ github.sha }}
+          assignees: PaulaCamargo25
+          delete-branch: true

build/loc/ImportTranslations.yml

@@ -0,0 +1,41 @@
+# ==================================================================================
+# Pipeline for VsCodeExtension-Localization build definition
+# Runs OneLocBuild task to localize xlf file
+# ==================================================================================
+
+resources:
+  repositories:
+  - repository: self
+    clean: true
+
+trigger: none
+pr: none
+schedules:
+- cron: "0 7 * * *"
+  displayName: Daily 7 AM
+  branches:
+    include:
+    - main
+  always: true
+
+pool:
+  name: 'AzurePipelines-EO'
+  demands:
+  - ImageOverride -equals AzurePipelinesWindows2022compliant
+variables:
+  VSCODE_EXTENSIONS_LOC_DIR: $(Pipeline.Workspace)/vscode-extensions-loc
+  EXTENSION_DIR: $(Build.SourcesDirectory)
+
+steps:
+# Clone extension Repository and run translations-export to get xlf file
+- script: git clone https://github.com/microsoft/vscode-extensions-loc/ ${VSCODE_EXTENSIONS_LOC_DIR}
+    displayName: Clone Microsoft/vscode-extensions-loc
+- task: CmdLine@2
+  inputs:
+    script: 'node ./translations_auto_pr.js microsoft vscode-python csigs csigs [email protected] "${Build.VSCODE_EXTENSIONS_LOC_DIR}/loc"'
+
+- task: PublishBuildArtifacts@1
+  inputs:
+    PathtoPublish: '$(Build.ArtifactStagingDirectory)'
+    ArtifactName: 'drop'
+    publishLocation: 'Container'

gulpfile.js

@@ -21,12 +21,19 @@ const flat = require('flat');
 const { argv } = require('yargs');
 const os = require('os');
 const rmrf = require('rimraf');
+const nls = require('vscode-nls-dev');
+const sourcemaps = require('gulp-sourcemaps');
+const typescript = require('typescript');
+const es = require('event-stream');
+const minimist = require('minimist');
+
+const tsProject = ts.createProject('./tsconfig.json', { typescript });
+const filter = require('gulp-filter');
 
 const isCI = process.env.TRAVIS === 'true' || process.env.TF_BUILD !== undefined;
 
 gulp.task('compile', (done) => {
     let failed = false;
-    const tsProject = ts.createProject('tsconfig.json');
     tsProject
         .src()
         .pipe(tsProject())
@@ -353,3 +360,120 @@ function hasNativeDependencies() {
     }
     return false;
 }
+
+const translationProjectName = 'ms-python.python';
+const translationExtensionName = 'vscode-python';
+
+const defaultLanguages = [
+    { id: 'de', folderName: 'de' },
+    { id: 'fr', folderName: 'fr' },
+    { id: 'es', folderName: 'es' },
+    { id: 'cs', folderName: 'cs' },
+    { id: 'it', folderName: 'it' },
+    { id: 'ja', folderName: 'ja' },
+    { id: 'ko', folderName: 'ko' },
+    { id: 'pl', folderName: 'pl' },
+    { id: 'pt-BR', folderName: 'pt-BR' },
+    { id: 'ru', folderName: 'ru' },
+    { id: 'tr', folderName: 'tr' },
+    { id: 'zh-Hans', folderName: 'zh-Hans' },
+    { id: 'zh-Hant', folderName: 'zh-Hant' },
+    { id: 'qps-ploc', folderName: 'qps-ploc' },
+];
+// ****************************
+// Command: translations-generate
+// The following is used to import an i18n directory structure and generate files used at runtime.
+// ****************************
+
+// Generate package.nls.*.json files from: ./i18n/*/package.i18n.json
+// Outputs to root path, as these nls files need to be along side package.json
+
+const generateAdditionalLocFiles = () =>
+    gulp
+        .src(['package.nls.json'])
+        .pipe(nls.createAdditionalLanguageFiles(defaultLanguages, 'i18n', 'out'))
+        .pipe(gulp.dest('out'));
+
+// Generates ./dist/nls.bundle.<language_id>.json from files in ./i18n/** *//<src_path>/<filename>.i18n.json
+// Localized strings are read from these files at runtime.
+const generateSrcLocBundle = () =>
+    // Transpile the TS to JS, and let vscode-nls-dev scan the files for calls to localize.
+    tsProject
+        .src()
+        .pipe(sourcemaps.init())
+        .pipe(tsProject())
+        .js.pipe(nls.createMetaDataFiles())
+        .pipe(nls.createAdditionalLanguageFiles(defaultLanguages, 'i18n', 'out'))
+        .pipe(nls.bundleMetaDataFiles('ms-vscode.python', 'out'))
+        .pipe(nls.bundleLanguageFiles())
+        .pipe(filter(['**/nls.bundle.*.json', '**/nls.metadata.header.json', '**/nls.metadata.json']))
+        .pipe(gulp.dest('out'));
+
+gulp.task('translations-generate', generateSrcLocBundle, generateAdditionalLocFiles);
+
+// ****************************
+// Command: translations-export
+// The following is used to export and XLF file containing english strings for translations.
+// The result will be written to: ../vscode-extensions-localization-export/ms-vscode/
+// ****************************
+const exportTranslations = (done) => {
+    const jsStream = tsProject.src().pipe(sourcemaps.init()).pipe(tsProject()).js.pipe(nls.createMetaDataFiles());
+
+    // Merge files from all source streams
+    jsStream
+
+        // Filter down to only the files we need
+        .pipe(filter(['**/*.nls.json', '**/*.nls.metadata.json']))
+
+        // Consoldate them into nls.metadata.json, which the xlf is built from.
+        .pipe(nls.bundleMetaDataFiles('ms-vscode.python', '.'))
+
+        // filter down to just the resulting metadata files
+        .pipe(filter(['**/nls.metadata.header.json', '**/nls.metadata.json']))
+
+        // Add package.nls.json, used to localized package.json
+        .pipe(gulp.src(['package.nls.json']))
+
+        // package.nls.json and nls.metadata.json are used to generate the xlf file
+        // Does not re-queue any files to the stream.  Outputs only the XLF file
+        .pipe(nls.createXlfFiles(translationProjectName, translationExtensionName))
+        .pipe(gulp.dest('../'))
+        .pipe(
+            es.wait(() => {
+                done();
+            }),
+        );
+};
+
+gulp.task('translations-export', exportTranslations);
+
+// ****************************
+// Command: translations-import
+// The following is used to import an XLF file containing all language strings.
+// This results in a i18n directory, which should be checked in.
+// ****************************
+
+// Imports translations from raw localized MLCP strings to VS Code .i18n.json files
+gulp.task('translations-import', (done) => {
+    const options = minimist(process.argv.slice(2), {
+        string: 'location',
+        default: {
+            location: '../vscode-translations-import',
+        },
+    });
+    es.merge(
+        defaultLanguages.map((language) => {
+            const id = language.transifexId || language.id;
+            return gulp
+                .src(path.join(options.location, id, `${translationExtensionName}.xlf`))
+                .pipe(nls.prepareJsonFiles())
+                .pipe(gulp.dest(path.join('./i18n', language.folderName)));
+        }),
+    ).pipe(
+        es.wait(() => {
+            done();
+        }),
+    );
+});
+
+gulp.task('create-i18n', gulp.series(generateSrcLocBundle, exportTranslations));

news/1 Enhancements/18286.md

@@ -0,0 +1 @@
+Enable localization using vscode-nlsß