[superseded] [WIP] fix #11457 csources2 revision is now version-controlled so we can keep updating boostrap nim version

.builds/freebsd.yml

@@ -21,9 +21,7 @@ tasks:
     sudo pkg install -y -q databases/sqlite3 devel/boehm-gc-threaded devel/pcre \
       devel/sdl20 devel/sfml www/node devel/gmake devel/git
     cd Nim
-    git clone --depth 1 -q https://github.com/nim-lang/csources.git
-    gmake -C csources -j $(sysctl -n hw.ncpu)
-    bin/nim c --skipUserCfg --skipParentCfg koch
+    NIMBUILD_ACTION=action_build_koch sh build_all.sh
     echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
 - test: |
     cd Nim

.github/workflows/ci_docs.yml

@@ -55,57 +55,61 @@ jobs:
         shell: bash
         run: echo "::add-path::${{ github.workspace }}/bin"
 
-      - name: 'Get current csources version'
-        id: csources-version
-        shell: bash
-        run: |
-          sha=$(git ls-remote https://github.com/nim-lang/csources master | cut -f 1)
-          echo "::set-output name=sha::$sha"
-
-      - name: 'Get prebuilt csources from cache'
-        id: csources-cache
-        uses: actions/cache@v1
-        with:
-          path: bin
-          key: '${{ matrix.os }}-${{ steps.csources-version.outputs.sha }}'
-
-      - name: 'Checkout csources'
-        if: steps.csources-cache.outputs.cache-hit != 'true'
-        uses: actions/checkout@v2
-        with:
-          repository: nim-lang/csources
-          path: csources
-
-      - name: 'Build 1-stage compiler from csources'
-        shell: bash
-        run: |
-          ext=
-          [[ '${{ runner.os }}' == 'Windows' ]] && ext=.exe
-          if [[ ! -x bin/nim-csources$ext ]]; then
-            ncpu=
-            case '${{ runner.os }}' in
-            'Linux')
-              ncpu=$(nproc)
-              ;;
-            'macOS')
-              ncpu=$(sysctl -n hw.ncpu)
-              ;;
-            'Windows')
-              ncpu=$NUMBER_OF_PROCESSORS
-              ;;
-            esac
-            [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
-
-            make -C csources -j $ncpu CC=gcc
-            cp bin/nim{,-csources}$ext
-          else
-            echo 'Cache hit, using prebuilt csources'
-            cp bin/nim{-csources,}$ext
-          fi
+      # - name: 'Get current csources version'
+      #   id: csources-version
+      #   shell: bash
+      #   run: |
+      #     sha=$(git ls-remote https://github.com/nim-lang/csources master | cut -f 1)
+      #     echo "::set-output name=sha::$sha"
+
+      # - name: 'Get prebuilt csources from cache'
+      #   id: csources-cache
+      #   uses: actions/cache@v1
+      #   with:
+      #     path: bin
+      #     key: '${{ matrix.os }}-${{ steps.csources-version.outputs.sha }}'
+
+      # - name: 'Checkout csources'
+      #   if: steps.csources-cache.outputs.cache-hit != 'true'
+      #   uses: actions/checkout@v2
+      #   with:
+      #     repository: nim-lang/csources
+      #     path: csources
+
+      # - name: 'Build 1-stage compiler from csources'
+      #   shell: bash
+      #   run: |
+      #     ext=
+      #     [[ '${{ runner.os }}' == 'Windows' ]] && ext=.exe
+      #     if [[ ! -x bin/nim-csources$ext ]]; then
+      #       ncpu=
+      #       case '${{ runner.os }}' in
+      #       'Linux')
+      #         ncpu=$(nproc)
+      #         ;;
+      #       'macOS')
+      #         ncpu=$(sysctl -n hw.ncpu)
+      #         ;;
+      #       'Windows')
+      #         ncpu=$NUMBER_OF_PROCESSORS
+      #         ;;
+      #       esac
+      #       [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+      #       make -C csources -j $ncpu CC=gcc
+      #       cp bin/nim{,-csources}$ext
+      #     else
+      #       echo 'Cache hit, using prebuilt csources'
+      #       cp bin/nim{-csources,}$ext
+      #     fi
+
+      # - name: 'Build koch'
+      #   shell: bash
+      #   run: nim c koch
 
       - name: 'Build koch'
         shell: bash
-        run: nim c koch
+        run: NIMBUILD_ACTION=action_build_koch sh build_all.sh
 
       - name: 'Build the real compiler'
         shell: bash

.github/workflows/ci_ssl.yml

@@ -1,14 +1,19 @@
 name: Nim SSL CI
+
 on:
+  push:
   pull_request:
     # Run only on changes on related files
     paths:
-      - 'lib/pure/httpclient.nim'
-      - 'lib/pure/net.nim'
-      - 'lib/pure/ssl_certs.nim'
-      - 'lib/wrappers/openssl.nim'
-      - 'tests/stdlib/thttpclient_ssl*'
-      - 'tests/untestable/thttpclient_ssl*'
+      # - 'lib/pure/httpclient.nim'
+      # - 'lib/pure/net.nim'
+      # - 'lib/pure/ssl_certs.nim'
+      # - 'lib/wrappers/openssl.nim'
+      # - 'tests/stdlib/thttpclient_ssl*'
+      # - 'tests/untestable/thttpclient_ssl*'
+      # - '**'
+      # - '.github/**'
+      - '.github/workflows/ci_ssl.yml'
 
 jobs:
   build:
@@ -23,12 +28,6 @@ jobs:
       - name: 'Checkout'
         uses: actions/checkout@v2
 
-      - name: 'Checkout csources'
-        uses: actions/checkout@v2
-        with:
-          repository: nim-lang/csources
-          path: csources
-
       - name: 'Install dependencies (Linux amd64)'
         if: runner.os == 'Linux' && matrix.cpu == 'amd64'
         run: |
@@ -53,28 +52,9 @@ jobs:
         shell: bash
         run: echo "::add-path::${{ github.workspace }}/bin"
 
-      - name: 'Build 1-stage compiler from csources'
-        shell: bash
-        run: |
-          ncpu=
-          case '${{ runner.os }}' in
-          'Linux')
-            ncpu=$(nproc)
-            ;;
-          'macOS')
-            ncpu=$(sysctl -n hw.ncpu)
-            ;;
-          'Windows')
-            ncpu=$NUMBER_OF_PROCESSORS
-            ;;
-          esac
-          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
-
-          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
-
-      - name: 'Build koch'
-        shell: bash
-        run: nim c koch
+      - name: 'Build csources => koch'
+        bash: NIMBUILD_ACTION=action_build_koch sh build_all.sh
+        displayName: 'Checkout csources'
 
       - name: 'Build the real compiler'
         shell: bash

azure-pipelines.yml

@@ -48,7 +48,7 @@ jobs:
 
     - checkout: self
 
-    - bash: git clone --depth 1 https://github.com/nim-lang/csources.git
+    - bash: NIMBUILD_ACTION=action_fetch_csources sh build_all.sh
       displayName: 'Checkout csources'
 
     - task: NodeTool@0

build_all.bat

@@ -1,7 +1,13 @@
 @echo off
-rem build development version of the compiler; can be rerun safely
+rem build development version of the compiler; can be rerun safely but won't pickup
+rem modifications in config/build_config.txt after 1st run, see build_all.sh for a way
+
+rem read in some common shared variables (shared with other tools)
+rem see https://stackoverflow.com/questions/3068929/how-to-read-file-contents-into-a-variable-in-a-batch-file
+for /f "delims== tokens=1,2" %%G in (config/build_config.txt) do set %%G=%%H
+
 if not exist csources (
-  git clone --depth 1 https://github.com/nim-lang/csources.git
+  git clone -q --depth 1 --branch %nim_csources2_tag% %nim_csources2url% csources
 )
 if not exist bin\nim.exe (
   cd csources
@@ -11,7 +17,7 @@ if not exist bin\nim.exe (
   CALL build.bat
   cd ..
 )
-bin\nim.exe c --skipUserCfg --skipParentCfg koch
-koch.exe boot -d:release --skipUserCfg --skipParentCfg
-koch.exe tools --skipUserCfg --skipParentCfg
+bin\nim.exe c --skipUserCfg --skipParentCfg --hints:off koch
+koch.exe boot -d:release --skipUserCfg --skipParentCfg --hints:off
+koch.exe tools --skipUserCfg --skipParentCfg --hints:off 
 

build_all.sh

@@ -1,7 +1,13 @@
 #! /bin/sh
 
-# build development version of the compiler; can be rerun safely.
+# build development version of the compiler; can be rerun safely and will pickup
+# any modification in config/build_config.txt
 # arguments can be passed, eg `--os freebsd`
+#
+# Usage:
+# sh build_all.sh # builds all
+# NIMBUILD_ACTION=action_build_koch sh build_all.sh # just builds everything up to koch
+
 
 set -u # error on undefined variables
 set -e # exit on first error
@@ -11,9 +17,20 @@ echo_run(){
   "$@"
 }
 
-[ -d csources ] || echo_run git clone -q --depth 1 https://github.com/nim-lang/csources.git
+echo_run . config/build_config.txt
+
+nim_csources=bin/nim_csources2
 
-nim_csources=bin/nim_csources
+fetch_nim_csources(){
+  (
+    [ -d csources ] || echo_run $nim_csources2_clone_cmd
+    echo_run cd csources
+    echo_run git remote set-url origin $nim_csources2url
+    echo_run git fetch -q --depth 1 origin tag $nim_csources2_tag
+    echo_run git checkout $nim_csources2_tag
+    echo_run git reset --hard $nim_csources2_tag
+  )
+}
 
 build_nim_csources_via_script(){
   echo_run cd csources
@@ -23,6 +40,8 @@ build_nim_csources_via_script(){
 build_nim_csources(){
   # avoid changing dir in case of failure
   (
+    [ -f bin/nim ] && echo_run rm bin/nim # otherwise wrongly says: `bin/nim' is up to date.
+    # it's cheap to redo the linking step anyway
     if [ $# -ne 0 ]; then
       # some args were passed (eg: `--cpu i386`), need to call build.sh
       build_nim_csources_via_script "$@"
@@ -37,16 +56,50 @@ build_nim_csources(){
       which $makeX && echo_run $makeX -C csources -j $((nCPU + 2)) -l $nCPU || build_nim_csources_via_script
     fi
   )
+  echo_run bin/nim -v
   # keep $nim_csources in case needed to investigate bootstrap issues
   # without having to rebuild from csources
   echo_run cp bin/nim $nim_csources
 }
 
-[ -f $nim_csources ] || echo_run build_nim_csources $@
+## stable API below here
+action_fetch_csources(){
+  echo_run fetch_nim_csources
+}
 
-# Note: if fails, may need to `cd csources && git pull`
-echo_run bin/nim c --skipUserCfg --skipParentCfg koch
+action_build_csources(){
+  action_fetch_csources
+  echo_run build_nim_csources
+}
+
+action_build_koch(){
+  action_build_csources
+  # always bootstrap from $nim_csources for reproducibility, in case this is rerun
+  echo_run $nim_csources c --skipUserCfg --skipParentCfg --hints:off koch
+}
+
+action_build_all(){
+  action_build_koch
+  # re-running without modifications takes 2 seconds up to this line
+  echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg --hints:off
+  echo_run ./koch tools --skipUserCfg --skipParentCfg --hints:off # Compile Nimble and other tools.
+}
 
-echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg
-echo_run ./koch tools --skipUserCfg --skipParentCfg # Compile Nimble and other tools.
+echo "NIMBUILD_ACTION: ${NIMBUILD_ACTION}"
 
+if [ -z "${NIMBUILD_ACTION}" ]; then
+  action_build_all # backward compatibility: same as action_build_all
+elif [ "${NIMBUILD_ACTION}" = "action_definitions" ]; then
+  echo "bash functions defined" # useful if we source this, then we can call individual functions
+elif [ "${NIMBUILD_ACTION}" = "action_fetch_csources" ]; then
+  action_fetch_csources
+elif [ "${NIMBUILD_ACTION}" = "action_build_csources" ]; then
+  echo_run fetch_nim_csources
+elif [ "${NIMBUILD_ACTION}" = "action_build_koch" ]; then
+  action_build_koch
+elif [ "${NIMBUILD_ACTION}" = "action_build_all" ]; then
+  action_build_all
+else
+  echo "unrecognized NIMBUILD_ACTION: $NIMBUILD_ACTION"
+  exit 1
+fi

ci/build.sh

@@ -1,15 +1,11 @@
 sh ci/deps.sh
 
 # Build from C sources.
-git clone --depth 1 https://github.com/nim-lang/csources.git
-cd csources
-sh build.sh
-cd ..
+NIMBUILD_ACTION=action_build_koch sh build_all.sh
+
 # Add Nim to the PATH
 export PATH=$(pwd)/bin${PATH:+:$PATH}
 # Bootstrap.
-nim -v
-nim c koch
 ./koch boot
 cp bin/nim bin/nimd
 ./koch boot -d:release

config/build_config.txt

@@ -0,0 +1,4 @@
+nim_comment="key-value pairs for windows/posix builds, locking down csources version (refs #11457)"
+nim_csources2url=https://github.com/timotheecour/csources
+nim_csources2_tag=v0.20.0
+nim_csources2_clone_cmd="git clone -q --depth 1 --branch $nim_csources2_tag $nim_csources2url csources"