From cabd8931e31c6621342b649e4a2a6708aaf7e2f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Fri, 27 Jul 2018 11:50:43 +0200 Subject: [PATCH 01/44] Fix internal fragmtn links in optimization.md --- src/content/configuration/optimization.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/content/configuration/optimization.md b/src/content/configuration/optimization.md index aed795601407..d163e8fb6e45 100644 --- a/src/content/configuration/optimization.md +++ b/src/content/configuration/optimization.md @@ -293,7 +293,7 @@ module.exports = { `bool` -Tells webpack to determine used exports for each module. This depends on [`optimization.providedExports`](#optimization-occurrenceorder). Information collected by `optimization.usedExports` is used by other optimizations or code generation i.e. exports are not generated for unused exports, export names are mangled to single char identifiers when all usages are compatible. +Tells webpack to determine used exports for each module. This depends on [`optimization.providedExports`](#optimizationoccurrenceorder). Information collected by `optimization.usedExports` is used by other optimizations or code generation i.e. exports are not generated for unused exports, export names are mangled to single char identifiers when all usages are compatible. Dead code elimination in minimizers will benefit from this and can remove unused exports. By default `optimization.usedExports` is enabled in `production` [mode](/concepts/mode/) and disabled elsewise. @@ -312,8 +312,8 @@ module.exports = { `bool` -Tells webpack to find segments of the module graph which can be safely concatenated into a single module. Depends on [`optimization.providedExports`](#optimization-providedexports) and [`optimization.usedExports`](#optimization-usedexports). -By default `optimization.concatenateModules` is enabled in `production` [mode](/concepts/mode/) and disabled elsewise. +Tells webpack to find segments of the module graph which can be safely concatenated into a single module. Depends on [`optimization.providedExports`](#optimizationprovidedexports) and [`optimization.usedExports`](#optimizationusedexports). +By default `optimization.concatenateModules` is enabled in `production` [mode](/concepts/mode/) and disabled elsewise. __webpack.config.js__ From 5321534e6f5ef4148fe2d61b37fe86e59c6b21e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Fri, 27 Jul 2018 12:06:24 +0200 Subject: [PATCH 02/44] Skip link checking or opencollective.com/webpack. Massive html response made the checker go into CPU overdrive --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bbc54ca9be4e..965aae8ebf78 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "lint:markdown": "markdownlint --config ./.markdownlint.json *.md ./src/content/**/*.md --ignore './src/content/**/_*.md'", "lint:social": "alex . -q", "lint:prose": "cp .proselintrc ~/ && proselint src/content", - "linkcheck": "hyperlink -r dist/index.html --canonicalroot https://webpack.js.org/ --skip support__ --skip sidecar.gitter.im --skip vimdoc.sourceforge.net --skip img.shields.io --skip npmjs.com/package/ --todo external-redirect | tee master.tap | tap-spot", + "linkcheck": "hyperlink -r dist/index.html --canonicalroot https://webpack.js.org/ --skip support__ --skip sidecar.gitter.im --skip vimdoc.sourceforge.net --skip img.shields.io --skip npmjs.com/package/ --skip opencollective.com/webpack --todo external-redirect | tee master.tap | tap-spot", "sitemap": "cd dist && sitemap-static --prefix=https://webpack.js.org/ > sitemap.xml", "serve": "npm run build && sirv start ./dist --port 4000", "deploy": "gh-pages -d dist" From 03e0dcffa2a9b9763d9a28383752b367e6f201cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Fri, 27 Jul 2018 16:48:56 +0200 Subject: [PATCH 03/44] Try out travis staged build --- .travis.yml | 53 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7770208100be..a3bf1cdbda42 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,44 @@ -branches: - only: - - master - - develop +sudo: required language: node_js node_js: - "8" -sudo: required -install: - - yarn - - sudo pip install proselint -before_script: - - source ./src/scripts/env.sh -script: - - bash ./src/scripts/deploy.sh + +cache: + yarn: true + pip: true + directories: + - ~/.proselint + +stages: + - lint + - build + - post-build + +jobs: + include: + - stage: lint + name: Linting + before_install: npm install --global yarn + install: yarn + script: npm run lint + cache: + directories: + - node_modules + + - stage: build + name: Build + before_script: + - source ./src/scripts/env.sh + script: npm run build + cache: + directories: + - dist + + - stage: post-build + name: Deploy + if: branch = master + script: bash ./src/scripts/deploy.sh + + - stage: post-build + name: Link check + script: npm run linkcheck From b31f42ed07c98b750c50fbf9a29976c1d5c8d0f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Fri, 27 Jul 2018 16:54:54 +0200 Subject: [PATCH 04/44] Add proselint --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index a3bf1cdbda42..cc5f666e8bcf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,8 @@ language: node_js node_js: - "8" +sudo: false + cache: yarn: true pip: true @@ -20,6 +22,8 @@ jobs: name: Linting before_install: npm install --global yarn install: yarn + before_script: + - pip install proselint script: npm run lint cache: directories: From e616a32242e186c6a1e84d882c160506ae5a1a84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 12:49:18 +0200 Subject: [PATCH 05/44] Upgrade pip before using it --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index cc5f666e8bcf..c9c6157b2155 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,7 @@ jobs: before_install: npm install --global yarn install: yarn before_script: + - pip install --upgrade pip - pip install proselint script: npm run lint cache: From 64d5ac9258a203ef57a1fd7505d565412bfecd79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 12:56:27 +0200 Subject: [PATCH 06/44] Move before-hooks around to try and make proselint install --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index c9c6157b2155..0d7af7cbd438 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,11 +20,11 @@ jobs: include: - stage: lint name: Linting - before_install: npm install --global yarn - install: yarn - before_script: + before_install: + - npm install --global yarn - pip install --upgrade pip - pip install proselint + install: yarn script: npm run lint cache: directories: From 012b5343adf8fc9b64e53e447dd16660b4c7ecf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 13:03:24 +0200 Subject: [PATCH 07/44] Try adding python as a language as well to geet an updated version --- .travis.yml | 5 +++-- requirements.txt | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 requirements.txt diff --git a/.travis.yml b/.travis.yml index 0d7af7cbd438..95590d51d2a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,8 @@ sudo: required language: node_js node_js: - "8" +python: + - "3.6" sudo: false @@ -22,8 +24,7 @@ jobs: name: Linting before_install: - npm install --global yarn - - pip install --upgrade pip - - pip install proselint + - pip install -r requirements.txt install: yarn script: npm run lint cache: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000000..6169a9005505 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +proselint From 3efa21d261abc6baf03f2e86863930f318c5acb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 13:09:06 +0200 Subject: [PATCH 08/44] More messing with config --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 95590d51d2a0..f1e9154a7d4c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,9 @@ -sudo: required -language: node_js -node_js: - - "8" +sudo: false +language: python python: - "3.6" +node_js: + - "8" sudo: false @@ -11,7 +11,7 @@ cache: yarn: true pip: true directories: - - ~/.proselint + - $HOME/.proselint stages: - lint From 2bcfb86f0f063c50851398727004c162f39d97ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 13:11:10 +0200 Subject: [PATCH 09/44] Manually handle node versioning --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f1e9154a7d4c..e92118aa38fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,6 @@ sudo: false language: python python: - "3.6" -node_js: - - "8" sudo: false @@ -23,6 +21,7 @@ jobs: - stage: lint name: Linting before_install: + - nvm use 8 - npm install --global yarn - pip install -r requirements.txt install: yarn From 10f18dfc518de7860c8a62a351f0c2ae4a54d0b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 13:13:49 +0200 Subject: [PATCH 10/44] Add node minor version to nvm install. Defaulted to slightly too low version --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e92118aa38fe..b7475dfd6166 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ jobs: - stage: lint name: Linting before_install: - - nvm use 8 + - nvm use 8.11 - npm install --global yarn - pip install -r requirements.txt install: yarn From ba535b5919f0cbca9bb17414d09a83c8dd8cef92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 13:17:28 +0200 Subject: [PATCH 11/44] Manually install node 8.11 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index b7475dfd6166..11466ccbcd2d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ cache: pip: true directories: - $HOME/.proselint + - $HOME/.nvm stages: - lint @@ -21,6 +22,7 @@ jobs: - stage: lint name: Linting before_install: + - nvm install 8.11 - nvm use 8.11 - npm install --global yarn - pip install -r requirements.txt From 315cdef10c3a033d65cde879010b7c50a171191f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 14:31:13 +0200 Subject: [PATCH 12/44] Try a matrix build to separate node and python stuff --- .travis.yml | 149 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 99 insertions(+), 50 deletions(-) diff --git a/.travis.yml b/.travis.yml index 11466ccbcd2d..dc7767e35cb7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,51 +1,100 @@ -sudo: false -language: python -python: - - "3.6" - -sudo: false - -cache: - yarn: true - pip: true - directories: - - $HOME/.proselint - - $HOME/.nvm - -stages: - - lint - - build - - post-build - -jobs: +# sudo: false +# language: python +# python: +# - "3.6" + +# sudo: false + +# cache: +# yarn: true +# pip: true +# directories: +# - "./node_modules" +# - $HOME/.proselint +# - $HOME/.nvm + +# stages: +# - lint +# - build +# - post-build + +# jobs: +# include: +# - stage: lint +# name: Linting +# before_install: +# - nvm install 8.11 +# - nvm use 8.11 +# - npm install --global yarn +# - pip install -r requirements.txt +# install: yarn +# script: npm run lint +# cache: +# directories: +# - node_modules + +# - stage: build +# name: Build +# before_script: +# - source ./src/scripts/env.sh +# script: npm run build +# cache: +# directories: +# - dist + +# - stage: post-build +# name: Deploy +# if: branch = master +# script: bash ./src/scripts/deploy.sh + +# - stage: post-build +# name: Link check +# script: npm run linkcheck + + + + + +######### + +sudo: required +dist: trusty + +matrix: include: - - stage: lint - name: Linting - before_install: - - nvm install 8.11 - - nvm use 8.11 - - npm install --global yarn - - pip install -r requirements.txt - install: yarn - script: npm run lint - cache: - directories: - - node_modules - - - stage: build - name: Build - before_script: - - source ./src/scripts/env.sh - script: npm run build - cache: - directories: - - dist - - - stage: post-build - name: Deploy - if: branch = master - script: bash ./src/scripts/deploy.sh - - - stage: post-build - name: Link check - script: npm run linkcheck + - language: python + python: + - 3.6 + + sudo: false + + cache: + pip: true + directories: + - $HOME/.proselint + + install: + - pip install -r requirements.txt + + script: + - cp .proselintrc ~/ && proselint src/content + + - language: node_js + node_js: + - "8" + + sudo: false + + cache: + yarn: true + + before_install: + - npm install --global yarn + + install: + - yarn + + script: + - yarn lint:js + - yarn lint:markdown + - yarn lint:social From ef4cafab32d8bf52d30611c06edafa597c00736f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 15:46:21 +0200 Subject: [PATCH 13/44] Add linkcheck task and try a different cahce setup --- .travis.yml | 51 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index dc7767e35cb7..db6e62400480 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,44 +57,65 @@ ######### -sudo: required -dist: trusty +cache: + yarn: true + pip: true + directories: + - $HOME/.proselint matrix: + allow_failures: + - name: Proselint + - name: Linkcheck + + fast_finish: true + include: - - language: python + - name: Linting + language: node_js + node_js: + - "8" + + sudo: false + + before_install: + - npm install --global yarn + + install: + - yarn + + script: + - yarn lint:js + - yarn lint:markdown + - yarn lint:social + + + - name: Proselint + language: python python: - 3.6 sudo: false - cache: - pip: true - directories: - - $HOME/.proselint - install: - pip install -r requirements.txt script: - cp .proselintrc ~/ && proselint src/content - - language: node_js + - name: Linkcheck + language: node_js node_js: - "8" sudo: false - cache: - yarn: true - before_install: - npm install --global yarn install: - yarn + - yarn build script: - - yarn lint:js - - yarn lint:markdown - - yarn lint:social + - yarn linkcheck From 1bfbcc8ecb42f627d3bc61bc68ea9407df98edea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 16:16:01 +0200 Subject: [PATCH 14/44] Minor name change to test if cache works correctly across builds --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index db6e62400480..35edf2a43437 100644 --- a/.travis.yml +++ b/.travis.yml @@ -71,7 +71,7 @@ matrix: fast_finish: true include: - - name: Linting + - name: Lintings language: node_js node_js: - "8" From f8a6cf8818071b4c07e9223199d68c2855a55a0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 17:21:10 +0200 Subject: [PATCH 15/44] Attempt to combine matrix and staged build --- .travis.yml | 71 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 35edf2a43437..2ba91203af4b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -71,24 +71,59 @@ matrix: fast_finish: true include: - - name: Lintings - language: node_js - node_js: - - "8" - - sudo: false - - before_install: - - npm install --global yarn - - install: - - yarn - - script: - - yarn lint:js - - yarn lint:markdown - - yarn lint:social - + - name: Build + + stages: + - lint + - build + - post-build + + jobs: + include: + - stage: lint + name: Linting + + before_install: + - npm install --global yarn + + install: yarn + + script: + - yarn lint:js + - yarn lint:markdown + - yarn lint:social + + cache: + directories: + - ./node_modules + + - stage: build + name: Build + script: + - source ./src/scripts/env.sh + - npm run build + cache: + directories: + - ./node_modules + - ./dist + + - stage: post-build + name: Deploy + if: branch = master + cache: + directories: + - ./node_modules + - ./dist + script: bash ./src/scripts/deploy.sh + + - stage: post-build + name: Link check + allow_failures: true + cache: + directories: + - ./node_modules + - ./dist + script: npm run linkcheck - name: Proselint language: python From df21ca2452bfe375911e22d6f8812d40c0f96022 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 17:38:56 +0200 Subject: [PATCH 16/44] Attempt going back to staged build --- .travis.yml | 283 ++++++++++++++++++++++++++-------------------------- 1 file changed, 144 insertions(+), 139 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2ba91203af4b..49236efeb9ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,55 +1,60 @@ -# sudo: false -# language: python -# python: -# - "3.6" +sudo: false +language: node_js +node_js: + - "8" -# sudo: false +cache: + yarn: true + pip: true + directories: + - "./node_modules + - "./dist + - "$HOME/.proselint" -# cache: -# yarn: true -# pip: true -# directories: -# - "./node_modules" -# - $HOME/.proselint -# - $HOME/.nvm +stages: + - smoke + - build + - post-build -# stages: -# - lint -# - build -# - post-build +jobs: + include: + - stage: lint + name: Linting + before_install: + - npm install --global yarn + install: yarn + script: yarn lint -# jobs: -# include: -# - stage: lint -# name: Linting -# before_install: -# - nvm install 8.11 -# - nvm use 8.11 -# - npm install --global yarn -# - pip install -r requirements.txt -# install: yarn -# script: npm run lint -# cache: -# directories: -# - node_modules + - stage: build + name: Build + before_script: + - source ./src/scripts/env.sh + script: yarn build + + - stage: post-build + name: Deploy + if: branch = master + script: bash ./src/scripts/deploy.sh -# - stage: build -# name: Build -# before_script: -# - source ./src/scripts/env.sh -# script: npm run build -# cache: -# directories: -# - dist + - stage: post-build + name: Link check + allow_failures: true + script: yarn linkcheck -# - stage: post-build -# name: Deploy -# if: branch = master -# script: bash ./src/scripts/deploy.sh + - stage: post-build + name: Proselint + language: python + python: + - 3.6 + allow_failures: true + before_install: + - pip install -r requirements.txt -# - stage: post-build -# name: Link check -# script: npm run linkcheck + install: + - pip install -r requirements.txt + + script: + - cp .proselintrc ~/ && proselint src/content @@ -57,100 +62,100 @@ ######### -cache: - yarn: true - pip: true - directories: - - $HOME/.proselint +# cache: +# yarn: true +# pip: true +# directories: +# - $HOME/.proselint -matrix: - allow_failures: - - name: Proselint - - name: Linkcheck +# matrix: +# allow_failures: +# - name: Proselint +# - name: Linkcheck - fast_finish: true +# fast_finish: true - include: - - name: Build - - stages: - - lint - - build - - post-build - - jobs: - include: - - stage: lint - name: Linting - - before_install: - - npm install --global yarn - - install: yarn - - script: - - yarn lint:js - - yarn lint:markdown - - yarn lint:social - - cache: - directories: - - ./node_modules - - - stage: build - name: Build - script: - - source ./src/scripts/env.sh - - npm run build - cache: - directories: - - ./node_modules - - ./dist - - - stage: post-build - name: Deploy - if: branch = master - cache: - directories: - - ./node_modules - - ./dist - script: bash ./src/scripts/deploy.sh - - - stage: post-build - name: Link check - allow_failures: true - cache: - directories: - - ./node_modules - - ./dist - script: npm run linkcheck - - - name: Proselint - language: python - python: - - 3.6 - - sudo: false - - install: - - pip install -r requirements.txt - - script: - - cp .proselintrc ~/ && proselint src/content - - - name: Linkcheck - language: node_js - node_js: - - "8" - - sudo: false - - before_install: - - npm install --global yarn - - install: - - yarn - - yarn build - - script: - - yarn linkcheck +# include: +# - name: Build + +# stages: +# - lint +# - build +# - post-build + +# jobs: +# include: +# - stage: lint +# name: Linting + +# before_install: +# - npm install --global yarn + +# install: yarn + +# script: +# - yarn lint:js +# - yarn lint:markdown +# - yarn lint:social + +# cache: +# directories: +# - ./node_modules + +# - stage: build +# name: Build +# script: +# - source ./src/scripts/env.sh +# - yarn build +# cache: +# directories: +# - ./node_modules +# - ./dist + +# - stage: post-build +# name: Deploy +# if: branch = master +# cache: +# directories: +# - ./node_modules +# - ./dist +# script: bash ./src/scripts/deploy.sh + +# - stage: post-build +# name: Link check +# allow_failures: true +# cache: +# directories: +# - ./node_modules +# - ./dist +# script: yarn linkcheck + +# - name: Proselint +# language: python +# python: +# - 3.6 + +# sudo: false + +# install: +# - pip install -r requirements.txt + +# script: +# - cp .proselintrc ~/ && proselint src/content + +# - name: Linkcheck +# language: node_js +# node_js: +# - "8" + +# sudo: false + +# before_install: +# - npm install --global yarn + +# install: +# - yarn +# - yarn build + +# script: +# - yarn linkcheck From b3440065f2d9d5bac517cf413d8a56b6b23c7168 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 17:50:07 +0200 Subject: [PATCH 17/44] Bump travis --- .travis.yml | 104 ---------------------------------------------------- 1 file changed, 104 deletions(-) diff --git a/.travis.yml b/.travis.yml index 49236efeb9ac..4edee2c53eba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,107 +55,3 @@ jobs: script: - cp .proselintrc ~/ && proselint src/content - - - - - -######### - -# cache: -# yarn: true -# pip: true -# directories: -# - $HOME/.proselint - -# matrix: -# allow_failures: -# - name: Proselint -# - name: Linkcheck - -# fast_finish: true - -# include: -# - name: Build - -# stages: -# - lint -# - build -# - post-build - -# jobs: -# include: -# - stage: lint -# name: Linting - -# before_install: -# - npm install --global yarn - -# install: yarn - -# script: -# - yarn lint:js -# - yarn lint:markdown -# - yarn lint:social - -# cache: -# directories: -# - ./node_modules - -# - stage: build -# name: Build -# script: -# - source ./src/scripts/env.sh -# - yarn build -# cache: -# directories: -# - ./node_modules -# - ./dist - -# - stage: post-build -# name: Deploy -# if: branch = master -# cache: -# directories: -# - ./node_modules -# - ./dist -# script: bash ./src/scripts/deploy.sh - -# - stage: post-build -# name: Link check -# allow_failures: true -# cache: -# directories: -# - ./node_modules -# - ./dist -# script: yarn linkcheck - -# - name: Proselint -# language: python -# python: -# - 3.6 - -# sudo: false - -# install: -# - pip install -r requirements.txt - -# script: -# - cp .proselintrc ~/ && proselint src/content - -# - name: Linkcheck -# language: node_js -# node_js: -# - "8" - -# sudo: false - -# before_install: -# - npm install --global yarn - -# install: -# - yarn -# - yarn build - -# script: -# - yarn linkcheck From 625c6e7d3a22fd024576e0e203d65c3a4d30eb4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 18:55:19 +0200 Subject: [PATCH 18/44] Ping Travis. You alive? --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 4edee2c53eba..46cb37511335 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ cache: - "./node_modules - "./dist - "$HOME/.proselint" + - "$HOME/foo" stages: - smoke From 3416ecb9b7710fd965ef99d8fb4777cf8b0d06f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 18:59:29 +0200 Subject: [PATCH 19/44] Fix broken travis.yml --- .travis.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 46cb37511335..9af57e1bce3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,10 +7,9 @@ cache: yarn: true pip: true directories: - - "./node_modules - - "./dist - - "$HOME/.proselint" - - "$HOME/foo" + - ./node_modules + - ./dist + - $HOME/.proselint stages: - smoke @@ -24,18 +23,21 @@ jobs: before_install: - npm install --global yarn install: yarn - script: yarn lint + script: + - yarn lint - stage: build name: Build before_script: - source ./src/scripts/env.sh - script: yarn build + script: + - yarn build - stage: post-build name: Deploy if: branch = master - script: bash ./src/scripts/deploy.sh + script: + - bash ./src/scripts/deploy.sh - stage: post-build name: Link check @@ -48,9 +50,6 @@ jobs: python: - 3.6 allow_failures: true - before_install: - - pip install -r requirements.txt - install: - pip install -r requirements.txt From 35459170b92e3c0c88bcb33f8b42ebb84a9833f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 19:01:14 +0200 Subject: [PATCH 20/44] Fix wrong stage order --- .travis.yml | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9af57e1bce3a..0a519b6f1063 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ cache: - $HOME/.proselint stages: - - smoke + - lint - build - post-build @@ -20,24 +20,19 @@ jobs: include: - stage: lint name: Linting - before_install: - - npm install --global yarn + before_install: npm install --global yarn install: yarn - script: - - yarn lint + script: yarn lint - stage: build name: Build - before_script: - - source ./src/scripts/env.sh - script: - - yarn build + before_script: source ./src/scripts/env.sh + script: yarn build - stage: post-build name: Deploy if: branch = master - script: - - bash ./src/scripts/deploy.sh + script: bash ./src/scripts/deploy.sh - stage: post-build name: Link check @@ -47,11 +42,7 @@ jobs: - stage: post-build name: Proselint language: python - python: - - 3.6 + python: 3.6 allow_failures: true - install: - - pip install -r requirements.txt - - script: - - cp .proselintrc ~/ && proselint src/content + install: pip install -r requirements.txt + script: cp .proselintrc ~/ && proselint src/content From 5d238f4f7afffb0f559c30b7afb21ddb8cbc923c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 19:07:11 +0200 Subject: [PATCH 21/44] Explicitly run specific lintings, exclude proselint --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0a519b6f1063..0fe2c23f43e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,10 @@ jobs: name: Linting before_install: npm install --global yarn install: yarn - script: yarn lint + script: + - yarn lint:js + - yarn lint:markdown + - yarn lint:social - stage: build name: Build From 2dc8216a3ecc976a1e865545b3ef313dd113c838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 20:52:45 +0200 Subject: [PATCH 22/44] Allow failures for link checker --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0fe2c23f43e3..4ea991591bbc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,10 @@ stages: - post-build jobs: + fast_finish: true + allow_failures: + - name: Link check + include: - stage: lint name: Linting @@ -39,13 +43,11 @@ jobs: - stage: post-build name: Link check - allow_failures: true script: yarn linkcheck - stage: post-build name: Proselint language: python python: 3.6 - allow_failures: true install: pip install -r requirements.txt script: cp .proselintrc ~/ && proselint src/content From d0c36fceb9f64e868af81aa80232798399eb39d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 21:15:56 +0200 Subject: [PATCH 23/44] Change proselint cache directory. Maybe this works --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4ea991591bbc..8324a9d6e8a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ cache: directories: - ./node_modules - ./dist - - $HOME/.proselint + - $HOME/.cache stages: - lint From 0c18717a7b91944f18972561e310114001f51f58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 21:57:46 +0200 Subject: [PATCH 24/44] Add new script to fetch repository names that the docs should contain. Try to centralize github API usage to avoid rate limitations --- package.json | 2 +- src/utilities/fetch-package-repos.js | 77 ++++++++++++++++++++++++++++ yarn.lock | 7 +-- 3 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 src/utilities/fetch-package-repos.js diff --git a/package.json b/package.json index 965aae8ebf78..866842defca8 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ ] }, "devDependencies": { - "@octokit/rest": "^15.2.6", + "@octokit/rest": "^15.9.4", "alex": "^5.1.0", "autoprefixer": "^7.2.3", "babel-core": "^6.26.0", diff --git a/src/utilities/fetch-package-repos.js b/src/utilities/fetch-package-repos.js new file mode 100644 index 000000000000..b54526d086c4 --- /dev/null +++ b/src/utilities/fetch-package-repos.js @@ -0,0 +1,77 @@ +const fs = require('fs'); +const path = require('path'); +const _ = require('lodash'); +const GithubAPI = require('@octokit/rest'); + +const fetch = { + loaders: [ + { + organization: 'webpack-contrib', + suffixes: ['-loader'] + }, + 'babel/babel-loader', + 'postcss/postcss-loader', + 'peerigon/extract-loader' + ], + + plugins: [ + { + organization: 'webpack-contrib', + suffixes: ['-webpack-plugin', '-extract-plugin'] + } + ] +}; + +const api = new GithubAPI(); + +async function paginate (org) { + let response = await api.repos.getForOrg({ org, type: 'public', per_page: 100}); + let {data} = response; + + while (api.hasNextPage(response)) { + response = await api.getNextPage(response); + data = data.concat(response.data); + } + + return data; +} + +async function main() { + if (process.env.GITHUB_TOKEN) { + api.authenticate({ + type: 'token', + token: process.env.GITHUB_TOKEN + }); + + const rateLimit = await api.misc.getRateLimit({}); + + console.log('Github API rate limit:', rateLimit.data.rate); + } + + for (const [type, collection] of Object.entries(fetch)) { + const result = await Promise.all(collection.map(async (item) => { + if (typeof item === 'string') { + return item; + } + + const { organization, suffixes } = item; + + const repos = await paginate(organization); + + return repos + .map(repo => repo.full_name) + .filter(name => suffixes.some(suffix => name.endsWith(suffix))); + })); + + const json = JSON.stringify(_.flatten(result), undefined, 2); + const jsonPath = path.resolve(__dirname, `../../repositories/${type}.json`); + + fs.writeFile(jsonPath, json, (err) => { + if (err) { + throw err; + } + }); + } +} + +main(); diff --git a/yarn.lock b/yarn.lock index f21a2682eb19..29be044ad7a3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8,7 +8,7 @@ dependencies: unist-util-visit "^1.3.0" -"@octokit/rest@^15.2.6": +"@octokit/rest@^15.9.4": version "15.9.4" resolved "https://registry.yarnpkg.com/@octokit/rest/-/rest-15.9.4.tgz#c6cf0f483275d9c798b18419b7c9d417493bb70f" dependencies: @@ -4454,12 +4454,13 @@ husky@^1.0.0-rc.8: version "3.0.1" resolved "https://codeload.github.com/Munter/hyperlink/tar.gz/070916d14ab8930a9184d545c38ed220863897bf" dependencies: - assetgraph "^4.5.0" + assetgraph "^4.0.5" async "^2.6.0" + lodash "^4.17.4" optimist "^0.6.1" pretty-bytes "^4.0.2" request "^2.83.0" - tap-render Munter/tap-render#0.1.7-patch4 + tap-render Munter/tap-render#0.1.7-patch1 urltools "^0.3.1" iconv-lite@0.4.19: From b88f90be660e53da08d97cb2f509be06ac9c2272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 22:09:37 +0200 Subject: [PATCH 25/44] Add caching to eslint --- .gitignore | 1 + .travis.yml | 5 +++-- package.json | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 9e4081fd86a7..b511fac6ff2c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ src/**/_*.md npm-debug.log yarn-error.log package-lock.json +.cache diff --git a/.travis.yml b/.travis.yml index 8324a9d6e8a9..8a532a07e470 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,9 @@ cache: yarn: true pip: true directories: - - ./node_modules - - ./dist + - node_modules + - dist + - .cache - $HOME/.cache stages: diff --git a/package.json b/package.json index 866842defca8..45b783dba428 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "build": "npm run fetch && npm run content && NODE_ENV=production webpack --config webpack.prod.js && npm run sitemap && echo webpack.js.org > dist/CNAME", "test": "npm run lint", "lint": "run-s lint:*", - "lint:js": "eslint src --ext .js,.jsx,.md", + "lint:js": "eslint src --ext .js,.jsx,.md --cache true --cache-location .cache/.eslintcache", "lint:markdown": "markdownlint --config ./.markdownlint.json *.md ./src/content/**/*.md --ignore './src/content/**/_*.md'", "lint:social": "alex . -q", "lint:prose": "cp .proselintrc ~/ && proselint src/content", From 08dbaef6e5ea2db476b8c7366acda362a1969725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 22:18:05 +0200 Subject: [PATCH 26/44] Remove parts of deploy.sh that are now run by travis --- src/scripts/deploy.sh | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/scripts/deploy.sh b/src/scripts/deploy.sh index 862fd2b2ca1e..072c68e742e1 100644 --- a/src/scripts/deploy.sh +++ b/src/scripts/deploy.sh @@ -2,26 +2,10 @@ # see https://gist.github.com/domenic/ec8b0fc8ab45f39403dd set -e # Exit with nonzero exit code if anything fails -SOURCE_BRANCH="master" - -# Pull requests and commits to other branches shouldn't try to deploy, just build to verify -if [ "$TRAVIS_PULL_REQUEST" != "false" -o "$TRAVIS_BRANCH" != "$SOURCE_BRANCH" ]; then - echo "Skipping deploy; just doing a build and linting links/prose/js." - yarn test - yarn build - exit 0 -fi - # Save some useful information REPO=`git config remote.origin.url` SSH_REPO=${REPO/https:\/\/github.com\//git@github.com:} -# Run tests -yarn test - -# Run our build -yarn build - # Set some git options git config --global user.name "Travis CI" git config --global user.email "ci@travis-ci.org" From 9f1a1cc7cc2928b8a05eda69f74ff851ff185a19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 22:49:32 +0200 Subject: [PATCH 27/44] Added new ./repositories to store github api results --- repositories/README.md | 3 ++ repositories/loaders.json | 45 ++++++++++++++++++++++++++++ repositories/plugins.json | 15 ++++++++++ src/utilities/fetch-package-repos.js | 3 ++ 4 files changed, 66 insertions(+) create mode 100644 repositories/README.md create mode 100644 repositories/loaders.json create mode 100644 repositories/plugins.json diff --git a/repositories/README.md b/repositories/README.md new file mode 100644 index 000000000000..ef2e81f5f1cb --- /dev/null +++ b/repositories/README.md @@ -0,0 +1,3 @@ +# Webpack documentation repositories + +The files in this directory are auto generated from `src/utils/fetch-package-repos.js` and should not be edited by hand. Any manual changes will be overwritten by the automation next time it runs. diff --git a/repositories/loaders.json b/repositories/loaders.json new file mode 100644 index 000000000000..a628da95edcf --- /dev/null +++ b/repositories/loaders.json @@ -0,0 +1,45 @@ +[ + "webpack-contrib/json-loader", + "webpack-contrib/raw-loader", + "webpack-contrib/coffee-loader", + "webpack-contrib/css-loader", + "webpack-contrib/style-loader", + "webpack-contrib/script-loader", + "webpack-contrib/less-loader", + "webpack-contrib/bundle-loader", + "webpack-contrib/val-loader", + "webpack-contrib/file-loader", + "webpack-contrib/url-loader", + "webpack-contrib/i18n-loader", + "webpack-contrib/json5-loader", + "webpack-contrib/worker-loader", + "webpack-contrib/jshint-loader", + "webpack-contrib/imports-loader", + "webpack-contrib/exports-loader", + "webpack-contrib/mocha-loader", + "webpack-contrib/coverjs-loader", + "webpack-contrib/expose-loader", + "webpack-contrib/node-loader", + "webpack-contrib/coffee-redux-loader", + "webpack-contrib/transform-loader", + "webpack-contrib/html-loader", + "webpack-contrib/sass-loader", + "webpack-contrib/source-map-loader", + "webpack-contrib/react-proxy-loader", + "webpack-contrib/null-loader", + "webpack-contrib/multi-loader", + "webpack-contrib/istanbul-instrumenter-loader", + "webpack-contrib/eslint-loader", + "webpack-contrib/yaml-frontmatter-loader", + "webpack-contrib/svg-inline-loader", + "webpack-contrib/restyle-loader", + "webpack-contrib/gzip-loader", + "webpack-contrib/cache-loader", + "webpack-contrib/thread-loader", + "webpack-contrib/polymer-webpack-loader", + "webpack-contrib/workerize-loader", + "webpack-contrib/config-loader", + "babel/babel-loader", + "postcss/postcss-loader", + "peerigon/extract-loader" +] \ No newline at end of file diff --git a/repositories/plugins.json b/repositories/plugins.json new file mode 100644 index 000000000000..5cf5a13c33a7 --- /dev/null +++ b/repositories/plugins.json @@ -0,0 +1,15 @@ +[ + "webpack-contrib/i18n-webpack-plugin", + "webpack-contrib/component-webpack-plugin", + "webpack-contrib/compression-webpack-plugin", + "webpack-contrib/extract-text-webpack-plugin", + "webpack-contrib/copy-webpack-plugin", + "webpack-contrib/npm-install-webpack-plugin", + "webpack-contrib/stylelint-webpack-plugin", + "webpack-contrib/babel-minify-webpack-plugin", + "webpack-contrib/uglifyjs-webpack-plugin", + "webpack-contrib/zopfli-webpack-plugin", + "webpack-contrib/closure-webpack-plugin", + "webpack-contrib/css-webpack-plugin", + "webpack-contrib/mini-css-extract-plugin" +] \ No newline at end of file diff --git a/src/utilities/fetch-package-repos.js b/src/utilities/fetch-package-repos.js index b54526d086c4..e7a18624fb24 100644 --- a/src/utilities/fetch-package-repos.js +++ b/src/utilities/fetch-package-repos.js @@ -1,5 +1,6 @@ const fs = require('fs'); const path = require('path'); +const mkdirp = require('mkdirp'); const _ = require('lodash'); const GithubAPI = require('@octokit/rest'); @@ -48,6 +49,8 @@ async function main() { console.log('Github API rate limit:', rateLimit.data.rate); } + mkdirp.sync(path.resolve(__dirname, `../../repositories/`)); + for (const [type, collection] of Object.entries(fetch)) { const result = await Promise.all(collection.map(async (item) => { if (typeof item === 'string') { From 7b9d41306444f3e4fd638fcf6d09c4e7e70321bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 23:33:49 +0200 Subject: [PATCH 28/44] Replace fetch.sh with npm scripts and fetch-package-readmes.js --- package.json | 6 ++- src/scripts/fetch.sh | 30 ------------ src/utilities/fetch-package-readmes.js | 66 ++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 31 deletions(-) delete mode 100644 src/scripts/fetch.sh create mode 100644 src/utilities/fetch-package-readmes.js diff --git a/package.json b/package.json index 45b783dba428..57baa1d4d334 100644 --- a/package.json +++ b/package.json @@ -26,9 +26,13 @@ "scripts": { "clean": "rimraf ./dist src/content/**/_*.md src/**/_*.json", "start": "cross-env NODE_ENV=development webpack-dev-server --config webpack.dev.js --env.dev", + "update-repos": "node src/utilities/fetch-package-repos.js", "content": "node src/scripts/build-content-tree.js ./src/content ./src/_content.json", "build-test": "npm run build && http-server build/", - "fetch": "sh src/scripts/fetch.sh", + "fetch": "run-p fetch:*", + "fetch:readmes": "node src/utilities/fetch-package-readmes.js", + "fetch:supporters": "node src/utilities/fetch-supporters.js", + "fetch:starter-kits": "node src/utilities/fetch-starter-kits.js", "prebuild": "npm run clean", "build": "npm run fetch && npm run content && NODE_ENV=production webpack --config webpack.prod.js && npm run sitemap && echo webpack.js.org > dist/CNAME", "test": "npm run lint", diff --git a/src/scripts/fetch.sh b/src/scripts/fetch.sh deleted file mode 100644 index a9b712051f44..000000000000 --- a/src/scripts/fetch.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -set -e # Exit with nonzero exit code if anything fails - -fetchPackages() { - # Fetch webpack-contrib (and various other) loader repositories - node ./src/utilities/fetch_packages.js "webpack-contrib" "-loader" "README.md" "./src/content/loaders" - node ./src/utilities/fetch_packages.js "babel" "babel-loader" "README.md" "./src/content/loaders" - node ./src/utilities/fetch_packages.js "postcss" "postcss-loader" "README.md" "./src/content/loaders" - node ./src/utilities/fetch_packages.js "peerigon" "extract-loader" "README.md" "./src/content/loaders" - - # Fetch webpack-contrib (and various other) plugin repositories - node ./src/utilities/fetch_packages.js "webpack-contrib" "-webpack-plugin" "README.md" "./src/content/plugins" - node ./src/utilities/fetch_packages.js "webpack-contrib" "-extract-plugin" "README.md" "./src/content/plugins" -} - -# If not defined, means running locally, so, fetch packages -if [ -z "$TRAVIS_PULL_REQUEST" ]; then - fetchPackages -# If defined and equal to false, means running in master, so, fetch packages -elif [ "$TRAVIS_PULL_REQUEST" = "false" ]; then - fetchPackages -else - echo "PR running, not fetching packages." -fi - -# Fetch sponsors and backers from opencollective -node ./src/utilities/fetch-supporters.js - -# Fetch starter kits -node ./src/utilities/fetch-starter-kits.js diff --git a/src/utilities/fetch-package-readmes.js b/src/utilities/fetch-package-readmes.js new file mode 100644 index 000000000000..8c7659c0f2ad --- /dev/null +++ b/src/utilities/fetch-package-readmes.js @@ -0,0 +1,66 @@ +const _ = require('lodash'); +const fs = require('fs'); +const path = require('path'); +const { promisify } = require('util'); +const mkdirp = promisify(require('mkdirp')); +const request = require('request-promise'); + +const yamlHeadmatter = require('./yaml-headmatter.js'); +const processReadme = require('./process-readme.js'); + +const writeFile = promisify(fs.writeFile); +const readFile = promisify(fs.readFile); +const cwd = process.cwd(); + +const types = ['loaders', 'plugins']; + +const pathMap = { + loaders: path.resolve(__dirname, '../content/loaders'), + plugins: path.resolve(__dirname, '../content/plugins') +}; + +async function main() { + for (const type of types) { + const outputDir = pathMap[type]; + + await mkdirp(outputDir); + + const repos = JSON.parse(await readFile(path.resolve(__dirname, `../../repositories/${type}.json`))); + + for (const repo of repos) { + const [org, packageName] = repo.split('/'); + const url = `https://raw.githubusercontent.com/${repo}/master/README.md`; + const htmlUrl = `https://github.com/${repo}/`; + const editUrl = `${htmlUrl}/edit/master/README.md`; + const fileName = path.resolve(outputDir, `_${packageName}.md`); + + let title = packageName; + + if (type === 'plugins') { + title = _.camelCase(title); + title = _.upperFirst(title); + title = title.replace(/I18N/, 'I18n'); + } + + // generate yaml matter for file + let headmatter = yamlHeadmatter({ + title: title, + source: url, + edit: editUrl, + repo: htmlUrl + }); + + request(url) + .then(async content => { + const body = processReadme(content, { source: url }); + + await writeFile(fileName, headmatter + body); + + console.log('Generated:', path.relative(cwd, fileName)); + }); + } + } +} + +main(); + From b2b63b60ae640b99ab99c1bc2a7285e45cc5387a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 23:42:39 +0200 Subject: [PATCH 29/44] Attempt to make caches more specific to the containers and stages they refer to --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8a532a07e470..24097fbc0063 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,12 +5,10 @@ node_js: cache: yarn: true - pip: true directories: - node_modules - dist - .cache - - $HOME/.cache stages: - lint @@ -50,5 +48,9 @@ jobs: name: Proselint language: python python: 3.6 + cache: + pip: true + directories: + - $HOME/.cache install: pip install -r requirements.txt script: cp .proselintrc ~/ && proselint src/content From 906455dffd368c64d596bac497eb1ac53d61c4cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sat, 28 Jul 2018 23:48:02 +0200 Subject: [PATCH 30/44] Remove deprecaed fetch_packages script. Now replaced by two-step fetch-package-repos and fetch-package-readmes --- src/utilities/fetch_packages.js | 158 -------------------------------- 1 file changed, 158 deletions(-) delete mode 100644 src/utilities/fetch_packages.js diff --git a/src/utilities/fetch_packages.js b/src/utilities/fetch_packages.js deleted file mode 100644 index a81f3b965f92..000000000000 --- a/src/utilities/fetch_packages.js +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env node -// ./fetch_package_names > output -// ./fetch_package_names "-loader" > output.json -const GitHubApi = require('@octokit/rest'); -const fs = require('fs'); -const urlModule = require('url'); -const path = require('path'); -const async = require('async'); -const mkdirp = require('mkdirp'); -const request = require('request-promise'); -const { promisify } = require('util'); -const _ = require('lodash'); - -const yamlHeadmatter = require('./yaml-headmatter.js'); -const processReadme = require('./process-readme.js'); - -const asyncWriteFile = promisify(fs.writeFile); - -if (require.main === module) { - main(); -} else { - module.exports = fetchPackageNames; -} - -function main() { - const organization = process.argv[2]; - const suffix = process.argv[3]; - const file = process.argv[4]; - const output = process.argv[5]; - - if (!organization) { - return console.error('Missing organization!'); - } - if (!suffix) { - return console.error('Missing suffix!'); - } - - if (!file) { - return console.error('Missing file!'); - } - - if (!output) { - return console.error('Missing output!'); - } - - mkdirp.sync(output); - - fetchPackageNames( - { - organization: organization, - suffix: suffix - }, - function(err, d) { - if (err) { - return console.error(err); - } - - fetchPackageFiles( - { - input: d, - file: file, - output: path.resolve(process.cwd(), output) - }, - function(err, d) { - if (err) { - return console.error(err); - } - - const msg = - d.length === 0 ? 'Fetched 0 files' : d.length === 1 ? 'Fetched 1 file: ' : `Fetched ${d.length} files: `; - console.log(msg + _.map(d, 'full_name')); - } - ); - } - ); -} - -function fetchPackageNames(options, cb) { - const github = new GitHubApi(); - - if (process.env.GITHUB_TOKEN) { - github.authenticate({ - type: 'token', - token: process.env.GITHUB_TOKEN - }); - - github.misc.getRateLimit({}, (err, res) => { - if (err) throw err; - console.log(res.data.rate); - }); - } - - // XXX: weak since this handles only one page - github.repos.getForOrg( - { - org: options.organization, - per_page: 100 - }, - function(err, d) { - if (err) { - return cb(err); - } - - return cb( - null, - d.data.filter(function(o) { - return o.name.endsWith(options.suffix); - }) - ); - } - ); -} - -function fetchPackageFiles(options, finalCb) { - const allPromises = options.input.map(pkg => { - // fetch from master branch - const branch = 'master'; - - // build fetch url - const file = options.file; - const baseUrl = 'https://raw.githubusercontent.com'; - const url = `${baseUrl}/${pkg.full_name}/${branch}/${file}`; - - return request(url) - .then(body => { - // modify README to fit page structure in site - if (body && file.toLowerCase() === 'readme.md') { - body = processReadme(body, { source: url }); - } - - let title = pkg.name; - - // process titles for plugins - if (title.match(/-plugin$/)) { - title = _.camelCase(title); - title = _.upperFirst(title); - title = title.replace(/I18N/, 'I18n'); - } - - // generate yaml matter for file - let headmatter = yamlHeadmatter({ - title: title, - source: url, - edit: `${pkg.html_url}/edit/${branch}/${file}`, - repo: pkg.html_url - }); - - return asyncWriteFile(path.resolve(options.output, `_${pkg.name}` + path.extname(file)), headmatter + body).then(done => { - return pkg; - }); - }) - .catch(error => { - console.log(error); - }); - }); - - return Promise.all(allPromises).then(data => finalCb(null, data)).catch(err => finalCb(err)); -} From 03c68df166b8d5bd06eb7b0cc498697217d75be4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sun, 29 Jul 2018 00:10:19 +0200 Subject: [PATCH 31/44] Attempt a more windows friendly build by using npm and webpack instead of shell commands --- package.json | 3 ++- src/CNAME | 1 + webpack.prod.js | 11 +++++++---- 3 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 src/CNAME diff --git a/package.json b/package.json index 57baa1d4d334..2dfe1097a4a8 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,8 @@ "fetch:supporters": "node src/utilities/fetch-supporters.js", "fetch:starter-kits": "node src/utilities/fetch-starter-kits.js", "prebuild": "npm run clean", - "build": "npm run fetch && npm run content && NODE_ENV=production webpack --config webpack.prod.js && npm run sitemap && echo webpack.js.org > dist/CNAME", + "build": "run-p fetch content && NODE_ENV=production webpack --config webpack.prod.js", + "postbuild": "npm run sitemap", "test": "npm run lint", "lint": "run-s lint:*", "lint:js": "eslint src --ext .js,.jsx,.md --cache true --cache-location .cache/.eslintcache", diff --git a/src/CNAME b/src/CNAME new file mode 100644 index 000000000000..39e36bc39ab8 --- /dev/null +++ b/src/CNAME @@ -0,0 +1 @@ +webpack.js.org diff --git a/webpack.prod.js b/webpack.prod.js index d3feb853b14b..f8f5425a42ce 100644 --- a/webpack.prod.js +++ b/webpack.prod.js @@ -30,10 +30,13 @@ const prod = { new webpack.DefinePlugin({ 'process.env.NODE_ENV': JSON.stringify('production') }), - new CopyWebpackPlugin([{ - from: './assets/icon-square-small-slack.png', - to: './assets/' - }]) + new CopyWebpackPlugin([ + { + from: './assets/icon-square-small-slack.png', + to: './assets/' + }, + 'CNAME' + ]) ] }; From 7d4bb064ec8a53fa2071cc4a38e0d2a45e6005e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sun, 29 Jul 2018 00:10:46 +0200 Subject: [PATCH 32/44] Disable link checking for now to speed up builds --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 24097fbc0063..cb5a83e1ebeb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,6 +41,7 @@ jobs: script: bash ./src/scripts/deploy.sh - stage: post-build + if: false # Temporarily shut off link checking to speed up builds name: Link check script: yarn linkcheck From b6b2476bd4d1230c8bd2ef772e7fcf4fdabcab08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sun, 29 Jul 2018 00:21:42 +0200 Subject: [PATCH 33/44] Revert "Disable link checking for now to speed up builds" This reverts commit 7d4bb064ec8a53fa2071cc4a38e0d2a45e6005e5. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cb5a83e1ebeb..24097fbc0063 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,7 +41,6 @@ jobs: script: bash ./src/scripts/deploy.sh - stage: post-build - if: false # Temporarily shut off link checking to speed up builds name: Link check script: yarn linkcheck From fc7676db10082bee1e466c7a7d978d1e6abf5899 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sun, 29 Jul 2018 00:39:53 +0200 Subject: [PATCH 34/44] Add dist to proselint cache so generated content also gets checked --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 24097fbc0063..fda41ecadfe3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,6 +51,7 @@ jobs: cache: pip: true directories: + - dist - $HOME/.cache install: pip install -r requirements.txt script: cp .proselintrc ~/ && proselint src/content From 993c998552d7a7d2a9606ddb2accd99146064c21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sun, 29 Jul 2018 00:55:23 +0200 Subject: [PATCH 35/44] Remove unnessessary GITHUB_TOKEN env variable check in repo fetching script --- src/utilities/fetch-package-repos.js | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/utilities/fetch-package-repos.js b/src/utilities/fetch-package-repos.js index e7a18624fb24..61ebeb0dce88 100644 --- a/src/utilities/fetch-package-repos.js +++ b/src/utilities/fetch-package-repos.js @@ -38,17 +38,6 @@ async function paginate (org) { } async function main() { - if (process.env.GITHUB_TOKEN) { - api.authenticate({ - type: 'token', - token: process.env.GITHUB_TOKEN - }); - - const rateLimit = await api.misc.getRateLimit({}); - - console.log('Github API rate limit:', rateLimit.data.rate); - } - mkdirp.sync(path.resolve(__dirname, `../../repositories/`)); for (const [type, collection] of Object.entries(fetch)) { From 8332fe3fc854976d7fe505f3d402393672bac572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sun, 29 Jul 2018 00:57:26 +0200 Subject: [PATCH 36/44] Revert "Add dist to proselint cache so generated content also gets checked" This reverts commit fc7676db10082bee1e466c7a7d978d1e6abf5899. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fda41ecadfe3..24097fbc0063 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,7 +51,6 @@ jobs: cache: pip: true directories: - - dist - $HOME/.cache install: pip install -r requirements.txt script: cp .proselintrc ~/ && proselint src/content From fbd7a2f894bec20c99efd6e19d017afc62124ddb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sun, 29 Jul 2018 08:16:29 +0200 Subject: [PATCH 37/44] Rework pipeline for better speed and make proselint a deployment blocker --- .travis.yml | 26 +++++++++++--------------- src/scripts/env.sh | 5 ----- 2 files changed, 11 insertions(+), 20 deletions(-) delete mode 100755 src/scripts/env.sh diff --git a/.travis.yml b/.travis.yml index 24097fbc0063..0e3b6d9f511a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,22 +29,9 @@ jobs: - yarn lint:js - yarn lint:markdown - yarn lint:social + - yarn build - - stage: build - name: Build - before_script: source ./src/scripts/env.sh - script: yarn build - - - stage: post-build - name: Deploy - if: branch = master - script: bash ./src/scripts/deploy.sh - - - stage: post-build - name: Link check - script: yarn linkcheck - - - stage: post-build + - stage: lint name: Proselint language: python python: 3.6 @@ -54,3 +41,12 @@ jobs: - $HOME/.cache install: pip install -r requirements.txt script: cp .proselintrc ~/ && proselint src/content + + - stage: post-build + name: Deploy + if: branch = master + script: bash ./src/scripts/deploy.sh + + - stage: post-build + name: Link check + script: yarn linkcheck diff --git a/src/scripts/env.sh b/src/scripts/env.sh deleted file mode 100755 index 41780cdcdeb9..000000000000 --- a/src/scripts/env.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set -e # Exit with nonzero exit code if anything fails - -export GITHUB_TOKEN=$GITHUB_TOKEN - From 65644a106d56917656d412a38b08eca4e63f35c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sun, 29 Jul 2018 08:19:14 +0200 Subject: [PATCH 38/44] Rename build stage to reflect its actions --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0e3b6d9f511a..32591e0ef792 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ jobs: include: - stage: lint - name: Linting + name: Lind and Build before_install: npm install --global yarn install: yarn script: From a2e614654aa3261fc36cac02b759ee0c54711d36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sun, 29 Jul 2018 09:04:33 +0200 Subject: [PATCH 39/44] Run content-tree only after generating all external content --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2dfe1097a4a8..7a9df7b0e39c 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "fetch:supporters": "node src/utilities/fetch-supporters.js", "fetch:starter-kits": "node src/utilities/fetch-starter-kits.js", "prebuild": "npm run clean", - "build": "run-p fetch content && NODE_ENV=production webpack --config webpack.prod.js", + "build": "run-s fetch content && NODE_ENV=production webpack --config webpack.prod.js", "postbuild": "npm run sitemap", "test": "npm run lint", "lint": "run-s lint:*", From 1ec6c28c8ec3e4767a1b546c4183e256f1c159a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sun, 29 Jul 2018 09:04:55 +0200 Subject: [PATCH 40/44] Remove link to non-existing named-modules-plugin --- src/content/plugins/commons-chunk-plugin.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/content/plugins/commons-chunk-plugin.md b/src/content/plugins/commons-chunk-plugin.md index 463d0c27aec9..28daea3b68aa 100644 --- a/src/content/plugins/commons-chunk-plugin.md +++ b/src/content/plugins/commons-chunk-plugin.md @@ -132,7 +132,7 @@ module.exports = { ``` -T> In combination with long term caching you may need to use the [`ChunkManifestWebpackPlugin`](https://github.com/soundcloud/chunk-manifest-webpack-plugin) to avoid the vendor chunk changes. You should also use records to ensure stable module ids, e.g. using [`NamedModulesPlugin`](/plugins/named-modules-plugin) or [`HashedModuleIdsPlugin`](/plugins/hashed-module-ids-plugin). +T> In combination with long term caching you may need to use the [`ChunkManifestWebpackPlugin`](https://github.com/soundcloud/chunk-manifest-webpack-plugin) to avoid the vendor chunk changes. You should also use records to ensure stable module ids, e.g. using `NamedModulesPlugin` or [`HashedModuleIdsPlugin`](/plugins/hashed-module-ids-plugin). ### Move common modules into the parent chunk From 80685a7ae25b5bc331030905df065c74dc031023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sun, 29 Jul 2018 09:05:17 +0200 Subject: [PATCH 41/44] Fix double slashes in edit links --- src/utilities/fetch-package-readmes.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utilities/fetch-package-readmes.js b/src/utilities/fetch-package-readmes.js index 8c7659c0f2ad..4eb68bf0bf24 100644 --- a/src/utilities/fetch-package-readmes.js +++ b/src/utilities/fetch-package-readmes.js @@ -30,7 +30,7 @@ async function main() { for (const repo of repos) { const [org, packageName] = repo.split('/'); const url = `https://raw.githubusercontent.com/${repo}/master/README.md`; - const htmlUrl = `https://github.com/${repo}/`; + const htmlUrl = `https://github.com/${repo}`; const editUrl = `${htmlUrl}/edit/master/README.md`; const fileName = path.resolve(outputDir, `_${packageName}.md`); From 42f0b1d55350343700fc3db1f510086795c3ef6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sun, 29 Jul 2018 09:32:14 +0200 Subject: [PATCH 42/44] Rename stages --- .travis.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 32591e0ef792..1edeb8f6c451 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,9 +11,8 @@ cache: - .cache stages: - - lint - - build - - post-build + - Build + - Post-build jobs: fast_finish: true @@ -21,8 +20,8 @@ jobs: - name: Link check include: - - stage: lint - name: Lind and Build + - stage: Build + name: Lint and Build before_install: npm install --global yarn install: yarn script: @@ -31,7 +30,7 @@ jobs: - yarn lint:social - yarn build - - stage: lint + - stage: Build name: Proselint language: python python: 3.6 @@ -42,11 +41,11 @@ jobs: install: pip install -r requirements.txt script: cp .proselintrc ~/ && proselint src/content - - stage: post-build + - stage: Post-build name: Deploy if: branch = master script: bash ./src/scripts/deploy.sh - - stage: post-build + - stage: Post-build name: Link check script: yarn linkcheck From dc053d925803c10f11671a58d2a621c28f6b7208 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sun, 29 Jul 2018 11:49:33 +0200 Subject: [PATCH 43/44] Add new internal link check as a blocking check --- .travis.yml | 3 ++- package.json | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1edeb8f6c451..9d92d083acd6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,7 @@ jobs: - yarn lint:markdown - yarn lint:social - yarn build + - yarn lint:links - stage: Build name: Proselint @@ -47,5 +48,5 @@ jobs: script: bash ./src/scripts/deploy.sh - stage: Post-build - name: Link check + name: External Link Check script: yarn linkcheck diff --git a/package.json b/package.json index 7a9df7b0e39c..b41c05ba5b75 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,8 @@ "lint:markdown": "markdownlint --config ./.markdownlint.json *.md ./src/content/**/*.md --ignore './src/content/**/_*.md'", "lint:social": "alex . -q", "lint:prose": "cp .proselintrc ~/ && proselint src/content", - "linkcheck": "hyperlink -r dist/index.html --canonicalroot https://webpack.js.org/ --skip support__ --skip sidecar.gitter.im --skip vimdoc.sourceforge.net --skip img.shields.io --skip npmjs.com/package/ --skip opencollective.com/webpack --todo external-redirect | tee master.tap | tap-spot", + "lint:links": "hyperlink -r dist/index.html --canonicalroot https://webpack.js.org/ -i | tee internal-links.tap | tap-spot", + "linkcheck": "hyperlink -r dist/index.html --canonicalroot https://webpack.js.org/ --skip support__ --skip sidecar.gitter.im --skip vimdoc.sourceforge.net --skip img.shields.io --skip npmjs.com/package/ --skip opencollective.com/webpack --todo external-redirect | tee external-links.tap | tap-spot", "sitemap": "cd dist && sitemap-static --prefix=https://webpack.js.org/ > sitemap.xml", "serve": "npm run build && sirv start ./dist --port 4000", "deploy": "gh-pages -d dist" @@ -91,7 +92,7 @@ "html-webpack-template": "^6.1.0", "http-server": "^0.10.0", "husky": "^1.0.0-rc.8", - "hyperlink": "github:Munter/hyperlink#070916d", + "hyperlink": "^4.0.0", "lint-staged": "^7.2.0", "loader-utils": "^1.1.0", "lodash": "^4.17.4", From ea7f606a7c484c3a686e94c0beb4d40f0525af83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20M=C3=BCller?= Date: Sun, 29 Jul 2018 12:03:39 +0200 Subject: [PATCH 44/44] Fix wrong name in allowed_failures config --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9d92d083acd6..0579a6ec7d3d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ stages: jobs: fast_finish: true allow_failures: - - name: Link check + - name: External Link Check include: - stage: Build