diff --git a/NodeBase/Dockerfile b/NodeBase/Dockerfile index 64da9493cd..54a033d405 100644 --- a/NodeBase/Dockerfile +++ b/NodeBase/Dockerfile @@ -178,8 +178,6 @@ ENV GENERATE_CONFIG true # A value higher than zero enables the feature ENV SE_DRAIN_AFTER_SESSION_COUNT 0 - - #======================== # Selenium Configuration #======================== diff --git a/Video/Dockerfile b/Video/Dockerfile index e69bddcd0e..455e6f047b 100644 --- a/Video/Dockerfile +++ b/Video/Dockerfile @@ -17,15 +17,15 @@ ENV DEBIAN_FRONTEND=noninteractive \ #======================== RUN apt-get -qqy update \ && apt-get -qqy --no-install-recommends install \ - supervisor x11-xserver-utils python3-pip \ + supervisor x11-xserver-utils python3-pip curl jq \ && rm -rf /var/lib/apt/lists/* /var/cache/apt/* #====================================== # Add Supervisor configuration files #====================================== COPY supervisord.conf /etc -COPY entry_point.sh video.sh video_ready.py /opt/bin/ -RUN cd /opt/bin && pip install psutil +COPY entry_point.sh video.sh video_ready.py start_uploader.sh /opt/bin/ +RUN cd /opt/bin && pip install psutil s3cmd RUN mkdir -p /var/run/supervisor /var/log/supervisor /videos diff --git a/Video/start_uploader.sh b/Video/start_uploader.sh new file mode 100755 index 0000000000..4fb0ad14c7 --- /dev/null +++ b/Video/start_uploader.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +if [ "${UPLOAD_TO_S3}" = "true" ]; +then + s3_location="${S3_VIDEOS_BUCKET}/$2" + echo "Uploading $1 $2 to S3 videos bucket ${S3_VIDEOS_BUCKET} $s3_location" + s3cmd --access_key="${AWS_ACCESS_KEY_ID}" --secret_key="${AWS_SECRET_ACCESS_KEY}" put $1 $s3_location +else + echo "Uploading to s3 disabled" +fi \ No newline at end of file diff --git a/Video/video.sh b/Video/video.sh index d764b726cf..d553210107 100755 --- a/Video/video.sh +++ b/Video/video.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash VIDEO_SIZE="${SE_SCREEN_WIDTH}""x""${SE_SCREEN_HEIGHT}" DISPLAY_CONTAINER_NAME=${DISPLAY_CONTAINER_NAME} @@ -22,6 +22,40 @@ until [ $return_code -eq 0 -o $attempts -eq $max_attempts ]; do attempts=$((attempts+1)) done -# exec replaces the video.sh process with ffmpeg, this makes easier to pass the process termination signal -exec ffmpeg -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} -i ${DISPLAY_CONTAINER_NAME}:${DISPLAY_NUM}.0 -codec:v ${CODEC} ${PRESET} -pix_fmt yuv420p "/videos/$FILE_NAME" - +video_location_default=/videos +if [ "${SESSION_VIDEO}" = "true" ]; +then + recording_started="false" + video_file_name="" + video_file="" + while true; + do + session_id=$(curl -s --request GET 'http://'${DISPLAY_CONTAINER_NAME:-localhost}':'${DISPLAY_CONTAINER_PORT}'/status' | jq -r '.[]?.node?.slots | .[0]?.session?.sessionId') + echo $session_id + if [ "$session_id" != "null" -a "$session_id" != "" ] && [ $recording_started = "false" ]; + then + echo "Starting to record video" + video_file_name="$session_id.mp4" + video_file="${VIDEO_LOCATION:-$video_location_default}/$video_file_name" + ffmpeg -nostdin -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} -i ${DISPLAY_CONTAINER_NAME}:${DISPLAY_NUM}.0 -codec:v ${CODEC} ${PRESET} -pix_fmt yuv420p $video_file & + recording_started="true" + echo "Video recording started" + elif [ "$session_id" = "null" -o "$session_id" = "" ] && [ $recording_started = "true" ]; + then + echo "Stopping to record video" + pkill --signal INT ffmpeg + /opt/bin/start_uploader.sh $video_file $video_file_name & + recording_started="false" + echo "Video recording stopped" + elif [ $recording_started = "true" ]; + then + echo "Video recording in progress" + else + echo "No session in progress" + fi + sleep 1 + done +else + # exec replaces the video.sh process with ffmpeg, this makes easier to pass the process termination signal + exec ffmpeg -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} -i ${DISPLAY_CONTAINER_NAME}:${DISPLAY_NUM}.0 -codec:v ${CODEC} ${PRESET} -pix_fmt yuv420p "${VIDEO_LOCATION:-$video_location_default}/$FILE_NAME" +fi \ No newline at end of file diff --git a/charts/selenium-grid/README.md b/charts/selenium-grid/README.md index 4f401587c5..a83bbc7db5 100644 --- a/charts/selenium-grid/README.md +++ b/charts/selenium-grid/README.md @@ -29,6 +29,12 @@ helm install selenium-grid docker-selenium/selenium-grid --version helm install selenium-grid --set ingress.hostname=selenium-grid.k8s.local docker-selenium/chart/selenium-grid/. ``` +## Enable Video Recording +To enable video recording you can set `RECORD_VIDEO` env variable in the browser node container to `"true"`. You can specify the location to store the recorded video using `VIDEO_LOCATION` env variable. The videos will have name format of `.mp4`. + +## Upload recorded videos to AWS S3 +If you have enabled video recording, you can also enable uploading them to S3 by setting `UPLOAD_TO_S3` env variable to `"true"`. You can specify the S3 bucket in 'S3_VIDEOS_BUCKET' env variable. You can specify your AWS credentials directly using the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and 'AWS_DEFAULT_REGION' env variables. If you use [IRSA](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) based authentication, you can specify the service account using `serviceAccount` variable in the charts. + ## Updating Selenium-Grid release Once you have a new chart version, you can update your selenium-grid running: @@ -95,6 +101,8 @@ This table contains the configuration parameters of the chart and their default | `chromeNode.lifecycle` | `{}` | hooks to make pod correctly shutdown or started | | `chromeNode.extraVolumeMounts` | `[]` | Extra mounts of declared ExtraVolumes into pod | | `chromeNode.extraVolumes` | `[]` | Extra Volumes declarations to be used in the pod (can be any supported volume type: ConfigMap, Secret, PVC, NFS, etc.) | +| `chromeNode.automountServiceAccountToken`| `false` | Determines automounting of service account token | +| `chromeNode.serviceAccount` | `` | Service account for chrome container | | `firefoxNode.enabled` | `true` | Enable firefox nodes | | `firefoxNode.deploymentEnabled` | `true` | Enable creation of Deployment for firefox nodes | | `firefoxNode.replicas` | `1` | Number of firefox nodes | @@ -123,6 +131,8 @@ This table contains the configuration parameters of the chart and their default | `firefoxNode.lifecycle` | `{}` | hooks to make pod correctly shutdown or started | | `firefoxNode.extraVolumeMounts` | `[]` | Extra mounts of declared ExtraVolumes into pod | | `firefoxNode.extraVolumes` | `[]` | Extra Volumes declarations to be used in the pod (can be any supported volume type: ConfigMap, Secret, PVC, NFS, etc.) | +| `firefoxNode.automountServiceAccountToken`| `false` | Determines automounting of service account token | +| `firefoxNode.serviceAccount` | `` | Service account for firefox container | | `edgeNode.enabled` | `true` | Enable edge nodes | | `edgeNode.deploymentEnabled` | `true` | Enable creation of Deployment for edge nodes | | `edgeNode.replicas` | `1` | Number of edge nodes | @@ -151,6 +161,8 @@ This table contains the configuration parameters of the chart and their default | `edgeNode.lifecycle` | `{}` | hooks to make pod correctly shutdown or started | | `edgeNode.extraVolumeMounts` | `[]` | Extra mounts of declared ExtraVolumes into pod | | `edgeNode.extraVolumes` | `[]` | Extra Volumes declarations to be used in the pod (can be any supported volume type: ConfigMap, Secret, PVC, NFS, etc.) | +| `edgeNode.automountServiceAccountToken`| `false` | Determines automounting of service account token | +| `edgeNode.serviceAccount` | `` | Service account for edge container | | `customLabels` | `{}` | Custom labels for k8s resources | diff --git a/charts/selenium-grid/templates/chrome-node-deployment.yaml b/charts/selenium-grid/templates/chrome-node-deployment.yaml index dcf5ea6f94..daf13a9156 100644 --- a/charts/selenium-grid/templates/chrome-node-deployment.yaml +++ b/charts/selenium-grid/templates/chrome-node-deployment.yaml @@ -29,6 +29,12 @@ spec: {{ toYaml . | nindent 8 }} {{- end }} spec: + {{- if .Values.chromeNode.automountServiceAccountToken }} + automountServiceAccountToken: {{ .Values.chromeNode.automountServiceAccountToken }} + {{- end }} + {{- if .Values.chromeNode.serviceAccount }} + serviceAccount: {{ .Values.chromeNode.serviceAccount }} + {{- end }} {{- with .Values.chromeNode.hostAliases }} hostAliases: {{ toYaml . | nindent 8 }} {{- end }} @@ -68,6 +74,37 @@ spec: {{- with .Values.chromeNode.startupProbe }} startupProbe: {{- toYaml . | nindent 12 }} {{- end }} + {{- if .Values.videoRecorder.chromeNode.enabled }} + - name: selenium-chrome-node-video + {{- $imageTag := .Values.videoRecorder.imageTag }} + image: {{ printf "%s:%s" .Values.videoRecorder.imageName $imageTag }} + imagePullPolicy: {{ .Values.videoRecorder.imagePullPolicy }} + {{- with .Values.videoRecorder.extraEnvironmentVariables }} + env: {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + envFrom: + - configMapRef: + name: {{ .Values.busConfigMap.name }} + {{- with .Values.videoRecorder.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if gt (len .Values.videoRecorder.ports) 0 }} + ports: + {{- range .Values.videoRecorder.ports }} + - containerPort: {{ . }} + protocol: TCP + {{- end }} + {{- end }} + volumeMounts: + - name: dshm + mountPath: /dev/shm + {{- if .Values.videoRecorder.extraVolumeMounts }} + {{- toYaml .Values.videoRecorder.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- with .Values.videoRecorder.resources }} + resources: {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} {{- if or .Values.global.seleniumGrid.imagePullSecret .Values.chromeNode.imagePullSecret }} imagePullSecrets: - name: {{ default .Values.global.seleniumGrid.imagePullSecret .Values.chromeNode.imagePullSecret }} diff --git a/charts/selenium-grid/templates/edge-node-deployment.yaml b/charts/selenium-grid/templates/edge-node-deployment.yaml index cb62346df3..e99523ac4e 100644 --- a/charts/selenium-grid/templates/edge-node-deployment.yaml +++ b/charts/selenium-grid/templates/edge-node-deployment.yaml @@ -29,6 +29,12 @@ spec: {{ toYaml . | nindent 8 }} {{- end }} spec: + {{- if .Values.edgeNode.automountServiceAccountToken }} + automountServiceAccountToken: {{ .Values.edgeNode.automountServiceAccountToken }} + {{- end }} + {{- if .Values.edgeNode.serviceAccount }} + serviceAccount: {{ .Values.edgeNode.serviceAccount }} + {{- end }} {{- with .Values.edgeNode.hostAliases }} hostAliases: {{ toYaml . | nindent 8 }} {{- end }} @@ -68,6 +74,37 @@ spec: {{- with .Values.edgeNode.startupProbe }} startupProbe: {{- toYaml . | nindent 12 }} {{- end }} + {{- if .Values.videoRecorder.edgeNode.enabled }} + - name: selenium-edge-node-video + {{- $imageTag := .Values.videoRecorder.imageTag }} + image: {{ printf "%s:%s" .Values.videoRecorder.imageName $imageTag }} + imagePullPolicy: {{ .Values.videoRecorder.imagePullPolicy }} + {{- with .Values.videoRecorder.extraEnvironmentVariables }} + env: {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + envFrom: + - configMapRef: + name: {{ .Values.busConfigMap.name }} + {{- with .Values.videoRecorder.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if gt (len .Values.videoRecorder.ports) 0 }} + ports: + {{- range .Values.videoRecorder.ports }} + - containerPort: {{ . }} + protocol: TCP + {{- end }} + {{- end }} + volumeMounts: + - name: dshm + mountPath: /dev/shm + {{- if .Values.videoRecorder.extraVolumeMounts }} + {{- toYaml .Values.videoRecorder.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- with .Values.videoRecorder.resources }} + resources: {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} {{- with .Values.edgeNode.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} diff --git a/charts/selenium-grid/templates/firefox-node-deployment.yaml b/charts/selenium-grid/templates/firefox-node-deployment.yaml index 9fdb8d8cf8..207c39605e 100644 --- a/charts/selenium-grid/templates/firefox-node-deployment.yaml +++ b/charts/selenium-grid/templates/firefox-node-deployment.yaml @@ -29,6 +29,12 @@ spec: {{ toYaml . | nindent 8 }} {{- end }} spec: + {{- if .Values.firefoxNode.automountServiceAccountToken }} + automountServiceAccountToken: {{ .Values.firefoxNode.automountServiceAccountToken }} + {{- end }} + {{- if .Values.firefoxNode.serviceAccount }} + serviceAccount: {{ .Values.firefoxNode.serviceAccount }} + {{- end }} {{- with .Values.firefoxNode.hostAliases }} hostAliases: {{ toYaml . | nindent 8 }} {{- end }} @@ -68,6 +74,37 @@ spec: {{- with .Values.firefoxNode.startupProbe }} startupProbe: {{- toYaml . | nindent 12 }} {{- end }} + {{- if .Values.videoRecorder.firefoxNode.enabled }} + - name: selenium-firefox-node-video + {{- $imageTag := .Values.videoRecorder.imageTag }} + image: {{ printf "%s:%s" .Values.videoRecorder.imageName $imageTag }} + imagePullPolicy: {{ .Values.videoRecorder.imagePullPolicy }} + {{- with .Values.videoRecorder.extraEnvironmentVariables }} + env: {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + envFrom: + - configMapRef: + name: {{ .Values.busConfigMap.name }} + {{- with .Values.videoRecorder.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if gt (len .Values.videoRecorder.ports) 0 }} + ports: + {{- range .Values.videoRecorder.ports }} + - containerPort: {{ . }} + protocol: TCP + {{- end }} + {{- end }} + volumeMounts: + - name: dshm + mountPath: /dev/shm + {{- if .Values.videoRecorder.extraVolumeMounts }} + {{- toYaml .Values.videoRecorder.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- with .Values.videoRecorder.resources }} + resources: {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} {{- with .Values.firefoxNode.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index 7b877894d9..f5e9852f53 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -341,6 +341,14 @@ chromeNode: # - "example.org" # Custom environment variables for chrome nodes extraEnvironmentVariables: + # - name: VIDEO_LOCATION + # value: /tmp + # - name: RECORD_VIDEO + # value: "false" + # - name: UPLOAD_TO_S3 + # value: "false" + # - name: S3_VIDEOS_BUCKET + # value: s3://selenium-grid/videos/ # - name: SE_JAVA_OPTS # value: "-Xmx512m" # - name: @@ -375,17 +383,17 @@ chromeNode: # failureThreshold: 120 # periodSeconds: 5 # Time to wait for pod termination - terminationGracePeriodSeconds: 30 + terminationGracePeriodSeconds: 3600 # Allow pod correctly shutdown - lifecycle: {} - # preStop: - # exec: - # command: - # - bash - # - -c - # - | - # curl -X POST 127.0.0.1:5555/se/grid/node/drain --header 'X-REGISTRATION-SECRET;' && \ - # while curl 127.0.0.1:5555/status; do sleep 1; done + lifecycle: + preStop: + exec: + command: + - bash + - -c + - | + curl -X POST 127.0.0.1:5555/se/grid/node/drain --header 'X-REGISTRATION-SECRET;' && \ + while curl 127.0.0.1:5555/status || pidof -x "start-uploader.sh" >/dev/null; do sleep 1; done; extraVolumeMounts: [] # - name: my-extra-volume @@ -398,6 +406,9 @@ chromeNode: # persistentVolumeClaim: # claimName: my-pv-claim + automountServiceAccountToken: false + serviceAccount: "" + # Configuration for firefox nodes firefoxNode: # Enable firefox nodes @@ -453,6 +464,14 @@ firefoxNode: # - "example.org" # Custom environment variables for firefox nodes extraEnvironmentVariables: + # - name: VIDEO_LOCATION + # value: /tmp + # - name: RECORD_VIDEO + # value: "false" + # - name: UPLOAD_TO_S3 + # value: "false" + # - name: S3_VIDEOS_BUCKET + # value: s3://selenium-grid/videos/ # - name: SE_JAVA_OPTS # value: "-Xmx512m" # - name: @@ -487,17 +506,17 @@ firefoxNode: # failureThreshold: 120 # periodSeconds: 5 # Time to wait for pod termination - terminationGracePeriodSeconds: 30 + terminationGracePeriodSeconds: 3600 # Allow pod correctly shutdown - lifecycle: {} - # preStop: - # exec: - # command: - # - bash - # - -c - # - | - # curl -X POST 127.0.0.1:5555/se/grid/node/drain --header 'X-REGISTRATION-SECRET;' && \ - # while curl 127.0.0.1:5555/status; do sleep 1; done + lifecycle: + preStop: + exec: + command: + - bash + - -c + - | + curl -X POST 127.0.0.1:5555/se/grid/node/drain --header 'X-REGISTRATION-SECRET;' && \ + while curl 127.0.0.1:5555/status || pidof -x "start-uploader.sh" >/dev/null; do sleep 1; done; extraVolumeMounts: [] # - name: my-extra-volume @@ -509,6 +528,9 @@ firefoxNode: # - name: my-extra-volume-from-pvc # persistentVolumeClaim: # claimName: my-pv-claim + automountServiceAccountToken: false + serviceAccount: "" + # Configuration for edge nodes edgeNode: @@ -564,8 +586,14 @@ edgeNode: # - "example.org" # Custom environment variables for edge nodes extraEnvironmentVariables: - # - name: SE_JAVA_OPTS - # value: "-Xmx512m" + # - name: VIDEO_LOCATION + # value: /tmp + # - name: RECORD_VIDEO + # value: "false" + # - name: UPLOAD_TO_S3 + # value: "false" + # - name: S3_VIDEOS_BUCKET + # value: s3://selenium-grid/videos/ # - name: # valueFrom: # secretKeyRef: @@ -599,18 +627,86 @@ edgeNode: # failureThreshold: 120 # periodSeconds: 5 # Time to wait for pod termination - terminationGracePeriodSeconds: 30 + terminationGracePeriodSeconds: 3600 # Allow pod correctly shutdown - lifecycle: {} - # preStop: - # exec: - # command: - # - bash - # - -c - # - | - # curl -X POST 127.0.0.1:5555/se/grid/node/drain --header 'X-REGISTRATION-SECRET;' && \ - # while curl 127.0.0.1:5555/status; do sleep 1; done + lifecycle: + preStop: + exec: + command: + - bash + - -c + - | + curl -X POST 127.0.0.1:5555/se/grid/node/drain --header 'X-REGISTRATION-SECRET;' && \ + while curl 127.0.0.1:5555/status || pidof -x "start-uploader.sh" >/dev/null; do sleep 1; done; + + extraVolumeMounts: [] + # - name: my-extra-volume + # mountPath: /home/seluser/Downloads + extraVolumes: [] + # - name: my-extra-volume + # emptyDir: {} + # - name: my-extra-volume-from-pvc + # persistentVolumeClaim: + # claimName: my-pv-claim + automountServiceAccountToken: false + serviceAccount: "" + +# Configuration for video recorder +videoRecorder: + # Image of video recorder + imageName: selenium/video + chromeNode: + enabled: false + firefoxNode: + enabled: false + edgeNode: + enabled: false + + # Image of video recorder + imageTag: ffmpeg-4.3.1-20230522 + # Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images) + imagePullPolicy: IfNotPresent + + ports: + - 5666 + resources: + requests: + memory: "1Gi" + cpu: "1" + limits: + memory: "1Gi" + cpu: "1" + extraEnvironmentVariables: + - name: VIDEO_LOCATION + value: /tmp + - name: UPLOAD_TO_S3 + value: "true" + - name: S3_VIDEOS_BUCKET + value: s3_bucket_name + - name: SESSION_VIDEO + value: "true" + - name: DISPLAY_CONTAINER_NAME + value: localhost + - name: DISPLAY_CONTAINER_PORT + value: "5900" + - name: AWS_ACCESS_KEY_ID + value: aws_access_key_id + - name: AWS_SECRET_ACCESS_KEY + value: aws_secret_access_key + # - name: + # valueFrom: + # secretKeyRef: + # name: secret-name + # key: secret-key + # Custom environment variables by sourcing entire configMap, Secret, etc. for video recorder. + extraEnvFrom: + # - configMapRef: + # name: proxy-settings + # - secretRef: + # name: mysecret + # Wait for pod startup + terminationGracePeriodSeconds: 3600 extraVolumeMounts: [] # - name: my-extra-volume # mountPath: /home/seluser/Downloads @@ -621,6 +717,8 @@ edgeNode: # - name: my-extra-volume-from-pvc # persistentVolumeClaim: # claimName: my-pv-claim + automountServiceAccountToken: false + serviceAccount: "" # Custom labels for k8s resources customLabels: {}