Merge pull request #1113 from alan-turing-institute/release-candidate-3.4.0

Release candidate 3.4.0

Author: jemrobinson

.PSScriptAnalyzerSettings.psd1

@@ -7,10 +7,6 @@
         # Ignore error when a function has both 'Username' and 'Password'
         # arguments
         "PSAvoidUsingUsernameAndPasswordParams",
-        # Ignore error around Write-Host. This is done to ensure that messages
-        # are actually logged to the console rather than forming part of the
-        # output of a function
-        "PSAvoidUsingWriteHost",
         # Ignore DSC errors as we want to phase out opaque DSC calls
         "PSDSCDscTestsPresent",
         "PSDSCDscExamplesPresent",

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,80 @@
+---
+name: Bug report
+about: Use this template for reporting a bug
+title: "<short description of issue>"
+labels: "type: bug"
+assignees: ""
+---
+
+<!--
+Before reporting a problem:
+- Make sure you are able to reproduce it on the [latest version](https://github.com/alan-turing-institute/data-safe-haven/releases)
+- Search the existing issues.
+- Refer to [our documentation](https://alan-turing-institute.github.io/data-safe-haven/).
+
+In order to help categorise your issue, please take the time to label it with at least one label from **each of** the following categories
+
+- type (bug; enhancement; documentation; question)
+- affected (developers; users)
+- severity (minor; medium; embarrassing; broken / unusable)
+
+Additionally, if you think this would be a good issue for newcomers to tackle then please add the `good first issue` label.
+
+:tada: Thank you!
+-->
+
+## :computer: System information
+
+- Operating System: <!-- Which operating system are you using: Windows, Linux, macOS etc. -->
+- Data Safe Haven version: <!-- Which version of the Data Safe Haven are you using: 3.0.0, 3.1.1 etc. -->
+- Powershell version: <!-- Which version of Powershell are you using: use CheckRequirements.ps1 if unsure -->
+- `Az` Powershell module version: <!-- Which version of the Az Powershell module are you using: use CheckRequirements.ps1  if unsure -->
+- `Microsoft.Graph` Powershell module version: <!-- Which version of the Microsoft.Graph Powershell module are you using: use CheckRequirements.ps1 if unsure -->
+- `Poshstache` module version: <!-- Which version of the Microsoft.Graph Powershell module are you using: use CheckRequirements.ps1 if unsure -->
+## :strawberry: Desired behaviour
+
+<!--
+A clear and concise description of what the behaviour should be when this issue is closed.
+
+Include specific tasks (if any) in the order in which they need to be done.
+- [ ] Task 1
+- [ ] Task 2
+- [ ] Task 3
+-->
+
+## :no_entry_sign: Actual behaviour
+
+<!--
+What happens now and (if relevant) an explanation of why this is incorrect
+-->
+
+## :recycle: To reproduce
+
+<!--
+How can the current behaviour be reproduced? Include some steps to reproduce the problem.
+
+Example:
+1. Deploy an SRE with option X set to Y
+2. Attempt to perform action Z
+3. Observe the following failure/error message
+-->
+
+<!-- OPTIONAL
+## :deciduous_tree: Log messages
+
+<details>
+<summary>First set of log messages</summary>
+
+```none
+Your log details here
+```
+</details>
+
+<details>
+<summary>Second set of log messages</summary>
+
+```none
+Your log details here
+```
+</details>
+-->

.github/ISSUE_TEMPLATE/enhancement_request.md

@@ -0,0 +1,40 @@
+---
+name: Enhancement request
+about: Use this template for requested new features or extensions to existing features
+title: "<short description of issue>"
+labels: "type: enhancement"
+assignees: ""
+---
+
+<!--
+Before opening an issue:
+- Make sure you are able to reproduce it on the [latest version](https://github.com/alan-turing-institute/data-safe-haven/releases)
+- Search the existing issues.
+- Refer to [our documentation](https://alan-turing-institute.github.io/data-safe-haven/).
+
+In order to help categorise your issue, please take the time to label it with at least one label from **each of** the following categories
+
+- type (bug; enhancement; documentation; question)
+- affected (developers; users)
+- severity (minor; medium; embarrassing; broken / unusable)
+
+Additionally, if you think this would be a good issue for newcomers to tackle then please add the `good first issue` label.
+
+:tada: Thank you!
+-->
+
+## :computer: System information
+
+- Operating System: <!-- Which operating system are you using: Windows, Linux, macOS etc. -->
+- Data Safe Haven version: <!-- Which version of the Data Safe Haven are you using: 3.0.0, 3.1.1 etc. -->
+
+## :strawberry: Desired behaviour
+
+<!--
+A clear and concise description of what the behaviour should be when this issue is closed.
+
+Include specific tasks (if any) in the order in which they need to be done.
+- [ ] Task 1
+- [ ] Task 2
+- [ ] Task 3
+-->

.github/issue_template.md

@@ -1,8 +1,10 @@
 <!--
-### :vertical_traffic_light: Your checklist for this pull request
-Please review the [guidelines for contributing](../CONTRIBUTING.md) to this repository.
+Thank you for your pull request!
+
+Before going any further, please review the [guidelines for contributing](../CONTRIBUTING.md) to this repository.
 
 - Make sure that you give your pull request a meaningful title (*e.g.* `Enable foobar integration` rather than `515 foobar`)
+- Make sure that you are targeting the **develop branch**
 - If this pull request is not ready to merge yet, please add `WIP:` to the beginning of the name as well as marking it as a **draft** pull request
 - Make sure that your branch is up-to-date with the **develop branch** (you probably started your branch from develop but it may have changed since then)
 - There is **no need** to use GitHub labels for pull requests. The issue(s) which this pull request is closing should have the appropriate labels.
@@ -12,24 +14,22 @@ Please review the [guidelines for contributing](../CONTRIBUTING.md) to this repo
 -->
 
 ### :arrow_heading_up: Summary
+
 <!--
 Please explain what your pull request does here.
+You might (optionally) want to include screenshots here.
 -->
 
 ### :closed_umbrella: Related issues
-<!--
-If your pull request will close any open issues (hopefully it will!) then add `Closes #<issue number>` here.
--->
 
-#### :camera: Screenshots
 <!--
-Optionally include screenshots here.
+If your pull request will close any open issues (hopefully it will!) then add `Closes #<issue number>` here.
 -->
 
 ### :microscope: Tests
+
 <!--
 - Document any manual tests that you have carried out (*e.g.* deploying a new SHM and/or SRE) and confirm which commit you did this with
 - Note that automated tests will be run as part of the CI process and will block this PR until they pass.
 - Additionally, a successful code review may be required before this PR can be merged.
 -->
-

.github/pull_request_template.md

@@ -0,0 +1,3 @@
+lxml
+natsort
+requests

.github/scripts/requirements.txt

@@ -0,0 +1,19 @@
+#! /usr/bin/env python3
+from lxml import html
+import hashlib
+import requests
+
+remote_page = requests.get("https://docs.microsoft.com/en-us/sql/azure-data-studio/download-azure-data-studio", allow_redirects=True)
+root = html.fromstring(remote_page.content)
+short_link = root.xpath("//a[contains(text(), '.deb file')]/@href")[0]
+
+remote_content = requests.get(short_link, allow_redirects=True)
+sha256 = hashlib.sha256(remote_content.content).hexdigest()
+version = remote_content.url.split("-")[-1].replace(".deb", "")
+remote = "/".join(remote_content.url.split("/")[:-1] + ["|DEBFILE|"])
+
+with open("deployment/secure_research_desktop/packages/deb-azuredatastudio.version", "w") as f_out:
+    f_out.write(f"hash: {sha256}\n")
+    f_out.write(f"version: {version}\n")
+    f_out.write("debfile: azuredatastudio-linux-|VERSION|.deb\n")
+    f_out.write(f"remote: {remote}\n")

.github/scripts/update_azure_data_studio.py

@@ -0,0 +1,33 @@
+#! /usr/bin/env python3
+import json
+from lxml import html
+from natsort import natsorted
+import requests
+
+output = {}
+remote_page = requests.get("https://repo1.maven.org/maven2/com/microsoft/sqlserver/mssql-jdbc/", allow_redirects=True)
+root = html.fromstring(remote_page.content)
+output["mssql_jdbc"] = natsorted([v for v in root.xpath("//a[contains(text(), 'jre8/')]/@href") if v != "../"])[-1].replace("/", "")
+
+remote_page = requests.get("https://repo1.maven.org/maven2/org/postgresql/pgjdbc-versions/", allow_redirects=True)
+root = html.fromstring(remote_page.content)
+output["pgjdbc"] = natsorted([v for v in root.xpath("//a[contains(text(), '/')]/@href") if v != "../"])[-1].replace("/", "")
+
+remote_page = requests.get("https://repo1.maven.org/maven2/org/postgresql/postgresql/", allow_redirects=True)
+root = html.fromstring(remote_page.content)
+output["postgresql"] = natsorted([v for v in root.xpath("//a[contains(text(), '/')]/@href") if v != "../"])[-1].replace("/", "")
+
+remote_page = requests.get("https://repo1.maven.org/maven2/net/postgis/postgis-jdbc/", allow_redirects=True)
+root = html.fromstring(remote_page.content)
+postgis_jdbc_versions = natsorted([v for v in root.xpath("//a[contains(text(), '/')]/@href") if v != "../"])
+
+remote_page = requests.get("https://repo1.maven.org/maven2/net/postgis/postgis-geometry/", allow_redirects=True)
+root = html.fromstring(remote_page.content)
+postgis_geometry_versions = natsorted([v for v in root.xpath("//a[contains(text(), '/')]/@href") if v != "../"])
+
+postgis = natsorted(set(postgis_jdbc_versions).intersection(set(postgis_geometry_versions)))[-1].replace("/", "")
+output["postgis_geometry"] = postgis
+output["postgis_jdbc"] = postgis
+
+with open("deployment/secure_research_desktop/packages/dbeaver-driver-versions.json", "w") as f_out:
+    f_out.writelines(json.dumps(output, indent=4, sort_keys=True))

.github/scripts/update_dbeaver_drivers.py

@@ -0,0 +1,19 @@
+#! /usr/bin/env python3
+from lxml import html
+import hashlib
+import requests
+
+remote_page = requests.get("https://www.rstudio.com/products/rstudio/download/", allow_redirects=True)
+root = html.fromstring(remote_page.content)
+short_link = [link for link in root.xpath("//a[contains(text(), '.deb')]/@href") if "debian" not in link][0]
+
+remote_content = requests.get(short_link, allow_redirects=True)
+sha256 = hashlib.sha256(remote_content.content).hexdigest()
+version = "-".join(remote_content.url.split("/")[-1].split("-")[1:-1])
+remote = "/".join(remote_content.url.split("/")[:-1] + ["|DEBFILE|"])
+
+with open("deployment/secure_research_desktop/packages/deb-rstudio.version", "w") as f_out:
+    f_out.write(f"hash: {sha256}\n")
+    f_out.write(f"version: {version}\n")
+    f_out.write("debfile: rstudio-|VERSION|-amd64.deb\n")
+    f_out.write(f"remote: {remote}\n")

.github/scripts/update_rstudio.py

@@ -0,0 +1,79 @@
+---
+name: Build allow lists
+
+# Run workflow on pushes to matching branches
+on:  # yamllint disable-line rule:truthy
+  push:
+    branches: [develop]
+  schedule:
+    - cron: "0 0 */6 * *"  # run every six days in order to keep the cache fresh
+
+# checkout needs 'contents:read'
+# pull request needs 'pull-requests:write' and 'contents:write'
+permissions:
+  contents: write
+  pull-requests: write
+
+env:
+  TIMEOUT_REACHED: 0
+
+jobs:
+  build_allow_lists:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Cache dependencies
+        uses: actions/cache@v2
+        with:
+          path: environment_configs/package_lists/dependency-cache.json
+          key: dependencies-${{ github.sha }}  # request a cache that does not yet exist
+          restore-keys: dependencies-          # use prefix matching to fallback to the most recently created SHA-specific cache
+
+      - name: Check PyPI allow list
+        shell: pwsh
+        env:
+          LIBRARIES_IO_API_KEY: ${{ secrets.LIBRARIES_IO_API_KEY }}
+        run: ./deployment/administration/SHM_Expand_Allowlist_Dependencies.ps1 -Repository pypi -TimeoutMinutes 60 -ApiKey "$env:LIBRARIES_IO_API_KEY"  # this will set TIMEOUT_REACHED if the timeout is reached
+
+      - name: Check CRAN allow list
+        shell: pwsh
+        env:
+          LIBRARIES_IO_API_KEY: ${{ secrets.LIBRARIES_IO_API_KEY }}
+        run: ./deployment/administration/SHM_Expand_Allowlist_Dependencies.ps1 -Repository cran -TimeoutMinutes 240 -ApiKey "$env:LIBRARIES_IO_API_KEY"  # this will set TIMEOUT_REACHED if the timeout is reached
+
+      - name: Check for changes
+        shell: bash
+        run: git --no-pager diff -- . ':!environment_configs/package_lists/dependency-cache.json'
+
+      - name: Get current date
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+
+      - name: Create pull request
+        if: ${{ (! env.TIMEOUT_REACHED) && (! env.ACT) }}
+        id: pull-request
+        uses: peter-evans/create-pull-request@dcd5fd746d53dd8de555c0f10bca6c35628be47a  # This commit corresponds to tag 3.12.0
+        with:
+          commit-message: Update PyPI and CRAN allow lists
+          committer: GitHub Actions <noreply@github.com>
+          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+          base: develop
+          branch: package-allowlist-updates
+          delete-branch: true
+          title: Update PyPI and CRAN allow lists
+          body: |
+            ### :arrow_heading_up: Summary
+            - Apply package allowlist diff from ${{ github.sha }} on ${{ steps.date.outputs.date }}
+
+            ### :closed_umbrella: Related issues
+            None
+
+            ### :microscope: Tests
+            Allow-list only
+          labels: |
+            affected: developers
+            severity: minor
+            type: enhancement
+          draft: false