move windows workflow to build ci and more

Author: microdev1

.github/workflows/build.yml

@@ -22,6 +22,7 @@ jobs:
     outputs:
       build-doc: ${{ steps.set-matrix.outputs.build-doc }}
       build-boards: ${{ steps.set-matrix.outputs.build-boards }}
+      build-windows: ${{ steps.set-matrix.outputs.build-windows }}
       boards-aarch: ${{ steps.set-matrix.outputs.boards-aarch }}
       boards-arm: ${{ steps.set-matrix.outputs.boards-arm }}
       boards-atmel: ${{ steps.set-matrix.outputs.boards-atmel }}
@@ -218,6 +219,74 @@ jobs:
         [ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi"
         [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*
 
+  build-windows:
+    runs-on: windows-2022
+    needs: scheduler
+    if: ${{ needs.scheduler.outputs.build-windows == 'True' }}
+    env:
+      CP_VERSION: ${{ needs.scheduler.outputs.cp-version }}
+    defaults:
+      run:
+        # We define a custom shell script here, although `msys2.cmd` does neither exist nor is it available in the PATH yet
+        shell: msys2 {0}
+    steps:
+    # We want to change the configuration of the git command that actions/checkout will be using
+    # (since it is not possible to set autocrlf through the action yet, see actions/checkout#226).
+    - run: git config --global core.autocrlf input
+      shell: bash
+    - name: Check python coding (cmd)
+      run: python -c "import sys, locale; print(sys.getdefaultencoding(), locale.getpreferredencoding(False))"
+      shell: cmd
+    # We use a JS Action, which calls the system terminal or other custom terminals directly, if required
+    - uses: msys2/setup-msys2@v2
+      with:
+        install: base-devel git wget unzip gcc python-pip
+    # The goal of this was to test how things worked when the default file encoding (locale.getpreferedencoding())
+    # was not UTF-8. However, msys2 python does use utf-8 as the preferred file encoding, and using actions/setup-python
+    # python3.8 gave a broken build, so we're not really testing what we wanted to test.
+    # However, commandline length limits are being tested so that does some good.
+    - name: Check python coding (msys2)
+      run: |
+        locale -v
+        which python; python --version
+        python -c "import sys, locale; print(sys.getdefaultencoding(), locale.getpreferredencoding(False))"
+        which python3; python3 --version
+        python3 -c "import sys, locale; print(sys.getdefaultencoding(), locale.getpreferredencoding(False))"
+    - name: Install dependencies
+      run: |
+        wget --no-verbose -O gcc-arm.zip https://developer.arm.com/-/media/Files/downloads/gnu-rm/10-2020q4/gcc-arm-none-eabi-10-2020-q4-major-win32.zip
+        unzip -q -d /tmp gcc-arm.zip
+        tar -C /tmp/gcc-arm-none-* -cf - . | tar -C /usr/local -xf -
+        pip install wheel
+        # requirements_dev.txt doesn't install on windows. (with msys2 python)
+        # instead, pick a subset for what we want to do
+        pip install cascadetoml jinja2 typer click intelhex
+        # check that installed packages work....?
+        which python; python --version; python -c "import cascadetoml"
+        which python3; python3 --version; python3 -c "import cascadetoml"
+    - name: Set up repository
+      uses: actions/checkout@v3
+      with:
+        submodules: false
+        fetch-depth: 1
+    - name: Set up submodules
+      uses: ./.github/actions/deps/submodules
+    - name: build mpy-cross
+      run: make -j2 -C mpy-cross
+    - name: build rp2040
+      run: make -j2 -C ports/raspberrypi BOARD=adafruit_feather_rp2040 TRANSLATION=de_DE
+    - name: build samd21
+      run: make -j2 -C ports/atmel-samd BOARD=feather_m0_express TRANSLATION=zh_Latn_pinyin
+    - name: build samd51
+      run: make -j2 -C ports/atmel-samd BOARD=feather_m4_express TRANSLATION=es
+    - name: build nrf
+      run: make -j2 -C ports/nrf BOARD=feather_nrf52840_express TRANSLATION=fr
+    - name: build stm
+      run: make -j2 -C ports/stm BOARD=feather_stm32f405_express TRANSLATION=pt_BR
+    # I gave up trying to do esp builds on windows when I saw
+    # ERROR: Platform MINGW64_NT-10.0-17763-x86_64 appears to be unsupported
+    # https://github.com/espressif/esp-idf/issues/7062
+
   aarch:
     needs: [scheduler, mpy-cross, tests]
     if: ${{ needs.scheduler.outputs.boards-aarch != '[]' }}

.github/workflows/ports_windows.yml

@@ -172,7 +172,7 @@ def get_bad_check_runs(query_check_runs):
 
     run_types = ["failed", "incomplete"]
 
-    regex_matrix = re.compile("^[^\n ]+ \/ (build|run) \([^\n ]+\)$")
+    regex_matrix = re.compile(r"^\S+ \/ (build|run) \(\S+\)$")
 
     while more_pages:
         check_runs = query_check_runs.fetch()["data"]["node"]

tools/ci_changes_per_commit.py

@@ -80,14 +80,16 @@ def main():
         submodules = ["extmod/ulab", "lib/", "tools/"]
     elif TARGET == "build-doc":
         # used in .readthedocs.yml to generate RTD
-        submodules = ["extmod/ulab", "frozen/"]
+        submodules = ["extmod/ulab"]
+        submodules_tags = ["frozen/"]
     elif TARGET == "mpy-cross" or TARGET == "mpy-cross-mac":
         submodules = ["tools/"]  # for huffman
-    elif TARGET == "windows":
+    elif TARGET == "build-windows":
         # This builds one board from a number of ports so fill out a bunch of submodules
         submodules = ["extmod/ulab", "lib/", "tools/", "ports/", "data/nvm.toml"]
     elif TARGET == "website":
-        submodules = ["tools/adabot/", "frozen/"]
+        submodules = ["tools/adabot/"]
+        submodules_tags = ["frozen/"]
     elif TARGET == "pre-commit":
         submodules = ["extmod/ulab"]
     else:

tools/ci_fetch_deps.py

@@ -62,6 +62,24 @@
 # Files in these directories never influence board builds
 IGNORE_DIRS = ["tests", "docs", ".devcontainer"]
 
+PATTERN_DOCS = (
+    r"^(?:\.github|docs|extmod\/ulab)|"
+    r"^(?:(?:ports\/\w+\/bindings|shared-bindings)\S+\.c|tools\/extract_pyi\.py|conf\.py|requirements-doc\.txt)$|"
+    r"(?:-stubs|\.(?:md|MD|rst|RST))$"
+)
+
+PATTERN_WINDOWS = [
+    ".github/",
+    "extmod/",
+    "lib/",
+    "mpy-cross/",
+    "ports/unix/",
+    "ports/windows/",
+    "py/",
+    "requirements",
+    "tools/",
+]
+
 if len(sys.argv) > 1:
     print("Using files list on commandline")
     changed_files = sys.argv[1:]
@@ -240,21 +258,19 @@ def set_docs_to_build(build_doc: bool):
         if last_failed_jobs.get("build-doc"):
             build_doc = True
         else:
-            doc_pattern = re.compile(
-                r"^(?:\.github\/workflows\/|docs|extmod\/ulab|(?:(?:ports\/\w+\/bindings|shared-bindings)\S+\.c|conf\.py|tools\/extract_pyi\.py|requirements-doc\.txt)$)|(?:-stubs|\.(?:md|MD|rst|RST))$"
-            )
+            doc_pattern = re.compile(PATTERN_DOCS)
             github_workspace = os.environ.get("GITHUB_WORKSPACE") or ""
             github_workspace = github_workspace and github_workspace + "/"
-            for p in changed_files:
-                if doc_pattern.search(p) and (
+            for file in changed_files:
+                if doc_pattern.search(file) and (
                     (
                         subprocess.run(
-                            f"git diff -U0 $BASE_SHA...$HEAD_SHA {github_workspace + p} | grep -o -m 1 '^[+-]\/\/|'",
+                            f"git diff -U0 $BASE_SHA...$HEAD_SHA {github_workspace + file} | grep -o -m 1 '^[+-]\/\/|'",
                             capture_output=True,
                             shell=True,
                         ).stdout
                     )
-                    if p.endswith(".c")
+                    if file.endswith(".c")
                     else True
                 ):
                     build_doc = True
@@ -265,6 +281,25 @@ def set_docs_to_build(build_doc: bool):
     set_output("build-doc", build_doc)
 
 
+def set_windows_to_build(build_windows):
+    if not build_windows:
+        if last_failed_jobs.get("build-windows"):
+            build_windows = True
+        else:
+            for file in changed_files:
+                for pattern in PATTERN_WINDOWS:
+                    if file.startswith(pattern):
+                        build_windows = True
+                        break
+                else:
+                    continue
+                break
+
+    # Set the step outputs
+    print("Building windows:", build_windows)
+    set_output("build-windows", build_windows)
+
+
 def check_changed_files():
     if not changed_files:
         print("Building all docs/boards")
@@ -277,6 +312,7 @@ def check_changed_files():
 def main():
     build_all = check_changed_files()
     set_docs_to_build(build_all)
+    set_windows_to_build(build_all)
     set_boards_to_build(build_all)