Merge develop to R-cran

.github/workflows/linting.yml

@@ -10,12 +10,12 @@ concurrency:
 
 jobs:
   python-linting:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
 
     - name: set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
       with:
         python-version: 3.8
 
@@ -28,4 +28,4 @@ jobs:
       run: flake8
 
     - name: Run flake8 to verify PEP8-compliance of Easyconfigs
-      run: flake8 --select F,W605  --filename '*.eb'
+      run: flake8 --select F,W605,W291  --filename '*.eb'

.github/workflows/tagbot.py

@@ -0,0 +1,182 @@
+# NOTE: In order to write comment and edit labels, this script requires workflows with write permissions.
+# It should not use any untrusted third party code, or any code checked into the repository itself
+# as that could indirectly grant PRs the ability to edit labels and comments on PRs.
+
+import os
+import git
+import requests
+import json
+from pathlib import Path
+
+
+def get_first_commit_date(repo, file_path):
+    commits = list(repo.iter_commits(paths=file_path))
+    if commits:
+        return commits[-1].committed_date
+    else:
+        raise ValueError(f'{file_path} has no commit info, this should not happen')
+
+
+def sort_by_added_date(repo, file_paths):
+    files_with_dates = [(get_first_commit_date(repo, file_path), file_path) for file_path in file_paths]
+    sorted_files = sorted(files_with_dates, reverse=True)
+    return [file for date, file in sorted_files]
+
+
+def similar_easyconfigs(repo, new_file):
+    possible_neighbours = [x for x in new_file.parent.glob('*.eb') if x != new_file]
+    return sort_by_added_date(repo, possible_neighbours)
+
+
+def pr_ecs(pr_diff):
+    new_ecs = []
+    changed_ecs = []
+    for item in pr_diff:
+        if item.a_path.endswith('.eb'):
+            if item.change_type == 'A':
+                new_ecs.append(Path(item.a_path))
+            else:
+                changed_ecs.append(Path(item.a_path))
+    return new_ecs, changed_ecs
+
+
+GITHUB_API_URL = 'https://api.github.com'
+event_path = os.getenv('GITHUB_EVENT_PATH')
+token = os.getenv('GH_TOKEN')
+repo = os.getenv('GITHUB_REPOSITORY')
+base_branch_name = os.getenv('GITHUB_BASE_REF')
+
+with open(event_path) as f:
+    data = json.load(f)
+
+pr_number = data['pull_request']['number']
+# Can't rely on merge_commit_sha for pull_request_target as it might be outdated
+# merge_commit_sha = data['pull_request']['merge_commit_sha']
+
+print("PR number:", pr_number)
+print("Base branch name:", base_branch_name)
+
+# Change into "pr" checkout directory to allow diffs and glob to work on the same content
+os.chdir('pr')
+gitrepo = git.Repo('.')
+
+target_commit = gitrepo.commit('origin/' + base_branch_name)
+print("Target commit ref:", target_commit)
+merge_commit = gitrepo.head.commit
+print("Merge commit:", merge_commit)
+pr_diff = target_commit.diff(merge_commit)
+
+new_ecs, changed_ecs = pr_ecs(pr_diff)
+modified_workflow = any(item.a_path.startswith('.github/workflows/') for item in pr_diff)
+
+
+print("Changed ECs:", ', '.join(str(p) for p in changed_ecs))
+print("Newly added ECs:", ', '.join(str(p) for p in new_ecs))
+print("Modified workflow:", modified_workflow)
+
+new_software = 0
+updated_software = 0
+to_diff = dict()
+for new_file in new_ecs:
+    neighbours = similar_easyconfigs(gitrepo, new_file)
+    print(f"Found {len(neighbours)} neighbours for {new_file}")
+    if neighbours:
+        updated_software += 1
+        to_diff[new_file] = neighbours
+    else:
+        new_software += 1
+
+print(f"Generating comment for {len(to_diff)} updates softwares")
+# Limit comment size for large PRs:
+if len(to_diff) > 20:  # Too much, either bad PR or some broad change. Not diffing.
+    max_diffs_per_software = 0
+elif len(to_diff) > 10:
+    max_diffs_per_software = 1
+elif len(to_diff) > 5:
+    max_diffs_per_software = 2
+else:
+    max_diffs_per_software = 3
+
+comment = ''
+if max_diffs_per_software > 0:
+    for new_file, neighbours in to_diff.items():
+        compare_neighbours = neighbours[:max_diffs_per_software]
+        if compare_neighbours:
+            print(f"Diffs for {new_file}")
+            comment += f'#### Updated software `{new_file.name}`\n\n'
+
+        for neighbour in compare_neighbours:
+            print(f"against {neighbour}")
+            comment += '<details>\n'
+            comment += f'<summary>Diff against <code>{neighbour.name}</code></summary>\n\n'
+            comment += f'[{neighbour}](https://github.com/{repo}/blob/{base_branch_name}/{neighbour})\n\n'
+            comment += '```diff\n'
+            comment += gitrepo.git.diff(f'HEAD:{neighbour}', f'HEAD:{new_file}')
+            comment += '\n```\n</details>\n\n'
+
+print("Adjusting labels")
+current_labels = [label['name'] for label in data['pull_request']['labels']]
+
+label_checks = [(changed_ecs, 'change'),
+                (new_software, 'new'),
+                (updated_software, 'update'),
+                (modified_workflow, 'workflow')]
+
+labels_add = []
+labels_del = []
+for condition, label in label_checks:
+    if condition and label not in current_labels:
+        labels_add.append(label)
+    elif not condition and label in current_labels:
+        labels_del.append(label)
+
+url = f"{GITHUB_API_URL}/repos/{repo}/issues/{pr_number}/labels"
+
+headers = {
+    "Accept": "application/vnd.github+json",
+    "Authorization": f"Bearer {token}",
+    "X-GitHub-Api-Version": "2022-11-28",
+}
+
+if labels_add:
+    print(f"Setting labels: {labels_add} at {url}")
+    response = requests.post(url, headers=headers, json={"labels": labels_add})
+    if response.status_code == 200:
+        print(f"Labels {labels_add} added successfully.")
+    else:
+        print(f"Failed to add labels: {response.status_code}, {response.text}")
+
+for label in labels_del:
+    print(f"Removing label: {label} at {url}")
+    response = requests.delete(f'{url}/{label}', headers=headers)
+    if response.status_code == 200:
+        print(f"Label {label} removed successfully.")
+    else:
+        print(f"Failed to delete label: {response.status_code}, {response.text}")
+
+# Write comment with diff
+if updated_software:
+    # Search for comment by bot to potentially replace
+    url = f"{GITHUB_API_URL}/repos/{repo}/issues/{pr_number}/comments"
+    response = requests.get(url, headers=headers)
+    comment_id = None
+    for existing_comment in response.json():
+        if existing_comment["user"]["login"] == "github-actions[bot]":  # Bot username in GitHub Actions
+            comment_id = existing_comment["id"]
+
+    if comment_id:
+        # Update existing comment
+        url = f"{GITHUB_API_URL}/repos/{repo}/issues/comments/{comment_id}"
+        response = requests.patch(url, headers=headers, json={"body": comment})
+        if response.status_code == 200:
+            print("Comment updated successfully.")
+        else:
+            print(f"Failed to update comment: {response.status_code}, {response.text}")
+    else:
+        # Post a new comment
+        url = f"{GITHUB_API_URL}/repos/{repo}/issues/{pr_number}/comments"
+        response = requests.post(url, headers=headers, json={"body": comment})
+        if response.status_code == 201:
+            print("Comment posted successfully.")
+        else:
+            print(f"Failed to post comment: {response.status_code}, {response.text}")

.github/workflows/tagbot.yml

@@ -0,0 +1,54 @@
+name: Tagbot
+on: [pull_request_target]
+
+concurrency:
+  group: "${{ github.workflow }}-${{ github.event.pull_request.number }}"
+  cancel-in-progress: true
+
+jobs:
+  tagbot:
+    # Note: can't rely on github.event.pull_request.merge_commit_sha because pull_request_target
+    # does not wait for github mergability check, and the value is outdated.
+    # Instead we merge manually in a temporary subdir "pr"
+    runs-on: ubuntu-24.04
+    permissions:
+      pull-requests: write
+    steps:
+    - name: Checkout base branch for workflow scripts
+      uses: actions/checkout@v4
+
+    - name: Checkout PR for computing diff into "pr" subdirectory
+      uses: actions/checkout@v4
+      with:
+        ref: "${{ github.event.pull_request.head.sha }}"
+        path: 'pr'
+        fetch-depth: 0
+
+    - name: Attempt test merge
+      id: merge
+      run: |
+        git config user.name "github-workflow"
+        git config user.email "[email protected]"
+        git merge --no-edit --no-ff origin/${{ github.event.pull_request.base.ref }}
+      continue-on-error: true
+      working-directory: pr
+
+    - name: Abort if merge failed
+      if: steps.merge.outcome == 'failure'
+      run: |
+        echo "Merge conflict detected, failing job."
+        exit 1
+
+    - name: set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.12
+
+    - name: Get packages
+      run: pip install gitpython requests
+
+    - name: Tag and comment
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: python .github/workflows/tagbot.py
+

.github/workflows/unit_tests.yml

@@ -10,31 +10,27 @@ concurrency:
 
 jobs:
   test-suite:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python: [3.6, '3.11']
-        modules_tool: [Lmod-7.8.22, Lmod-8.6.8]
+        python: ['3.7', '3.11']
+        modules_tool: [Lmod-8.6.8]
         module_syntax: [Lua, Tcl]
-        # exclude some configurations: only test Tcl module syntax with Lmod 8.x and Python 3.6
-        exclude:
-          - modules_tool: Lmod-7.8.22
-            module_syntax: Tcl
       fail-fast: false
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
       with:
         fetch-depth: 0 # Required for git merge-base to work
 
     - name: Cache source files in /tmp/sources
       id: cache-sources
-      uses: actions/cache@v2
+      uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # 4.2.2
       with:
         path: /tmp/sources
         key: eb-sourcepath
 
     - name: set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
       with:
         python-version: ${{matrix.python}}
         architecture: x64
@@ -46,23 +42,18 @@ jobs:
         # sudo apt-get update
         # for modules tool
         sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
-        # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
-        # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
-        if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
-            sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
-        fi
         # for testing OpenMPI-system*eb we need to have Open MPI installed
         sudo apt-get install libopenmpi-dev openmpi-bin
         # required for test_dep_graph
-        pip install pep8 python-graph-core python-graph-dot
+        pip install pycodestyle python-graph-core python-graph-dot
 
     - name: install EasyBuild framework
       run: |
           cd $HOME
           # first determine which branch of easybuild-framework repo to install
           BRANCH=develop
           if [ "x$GITHUB_BASE_REF" = 'xmain' ]; then BRANCH=main; fi
-          if [ "x$GITHUB_BASE_REF" = 'x4.x' ]; then BRANCH=4.x; fi
+          if [ "x$GITHUB_BASE_REF" = 'x5.0.x' ]; then BRANCH=5.0.x; fi
           echo "Using easybuild-framework branch $BRANCH (\$GITHUB_BASE_REF $GITHUB_BASE_REF)"
 
           git clone -b $BRANCH --depth 10 --single-branch https://github.com/easybuilders/easybuild-framework.git
@@ -139,28 +130,28 @@ jobs:
           grep "^robot-paths .*/easybuild/easyconfigs" eb_show_config.out
 
           # check whether some specific easyconfig files are found
-          echo "eb --search 'TensorFlow-1.14.*.eb'"
-          eb --search 'TensorFlow-1.14.*.eb' | tee eb_search_TF.out
-          grep '/TensorFlow-1.14.0-foss-2019a-Python-3.7.2.eb$' eb_search_TF.out
+          echo "eb --search 'TensorFlow-2.13.*.eb'"
+          eb --search 'TensorFlow-2.13.*.eb' | tee eb_search_TF.out
+          grep '/TensorFlow-2.13.0-foss-2023a.eb$' eb_search_TF.out
 
-          echo "eb --search '^foss-2019b.eb'"
-          eb --search '^foss-2019b.eb' | tee eb_search_foss.out
-          grep '/foss-2019b.eb$' eb_search_foss.out
+          echo "eb --search '^foss-2023a.eb'"
+          eb --search '^foss-2023a.eb' | tee eb_search_foss.out
+          grep '/foss-2023a.eb$' eb_search_foss.out
 
           # try installing M4 with system toolchain (requires ConfigureMake easyblock + easyconfig)
           # use /tmp/sources because that has cached downloads (see cache step above)
           eb --prefix /tmp/$USER/$GITHUB_SHA --sourcepath /tmp/sources M4-1.4.18.eb
 
   test-sdist:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python: [3.6, '3.11']
+        python: [3.7, '3.11']
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
 
     - name: set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
       with:
         python-version: ${{matrix.python}}
 

.gitignore

@@ -1,12 +1,15 @@
 .pydevproject
 .project
 LICENSE_HEADER
+*.eb.bak_*
 *.pyc
 *.pyo
 *.nja
+*.out
 build/
 dist/
 *egg-info/
+.venv/
 *.swp
 *.ropeproject/
 eb-*.log