Merge upstream

Author: Juliya Smith

.circleci/config.yml

@@ -3,12 +3,11 @@ jobs:
   build:
     docker:
       - image: devdemisto/content-build:3.0.0.7332  # disable-secrets-detection
-    resource_class: medium+
     parallelism: 2
     environment:
-      CONTENT_VERSION: "20.5.3"
+      CONTENT_VERSION: "20.6.0"
       SERVER_VERSION: "5.5.0"
-      GIT_SHA1: "bb4ac8b7943a6cba0c22935ebad6b6e17939840d" # guardrails-disable-line disable-secrets-detection
+      GIT_SHA1: "63646dff0fba977f91d6d9fc2d7fd233bfb5561b" # guardrails-disable-line disable-secrets-detection
     steps:
       - checkout
       - setup_remote_docker
@@ -88,6 +87,17 @@ jobs:
             else
                 echo ".circleci/config.yml is up to date!"
             fi
+      - run:
+          name: Update Tests step
+          when: always
+          command: |
+            if [ $CIRCLE_NODE_INDEX -ne 0 ] ;
+              then
+                echo "Skipping - running only in container number 0"
+                exit 0
+            fi
+            python3 ./Tests/scripts/update_conf_json.py
+            cp "./Tests/conf.json" "$CIRCLE_ARTIFACTS/conf.json"
       - run:
           name: Create ID Set
           when: always
@@ -116,9 +126,9 @@ jobs:
           name: Validate Files and Yaml
           when: always
           command: |
-            if [ -n "${INSTANCE_TESTS}" ] || [ $CIRCLE_NODE_INDEX -ne 0 ] ;
+            if [ -n "${INSTANCE_TESTS}" ] || [ $CIRCLE_NODE_INDEX -ne 0 ] || [ -n "${NIGHTLY}" ];
               then
-                echo "Skipping - not running in INSTANCE_TESTS build or unit-tests container"
+                echo "Skipping - not running in INSTANCE_TESTS build or unit-tests container or Nightly run"
                 exit 0
             fi
 
@@ -171,10 +181,14 @@ jobs:
             fi
             if [ -n "${GITHUB_TOKEN}" ] ;
               then
-                python3 release_notes.py $CONTENT_VERSION $GIT_SHA1 $CIRCLE_BUILD_NUM $SERVER_VERSION --github-token $GITHUB_TOKEN
+                # python3 release_notes.py $CONTENT_VERSION $GIT_SHA1 $CIRCLE_BUILD_NUM $SERVER_VERSION --github-token $GITHUB_TOKEN || echo "ignore errors"
+                # new release notes summary generator in packs format
+                python3 Utils/release_notes_generator.py $CONTENT_VERSION $GIT_SHA1 $CIRCLE_BUILD_NUM --output $CIRCLE_ARTIFACTS/packs-release-notes.md --github-token $GITHUB_TOKEN
 
               else
-                python3 release_notes.py $CONTENT_VERSION $GIT_SHA1 $CIRCLE_BUILD_NUM $SERVER_VERSION
+                # python3 release_notes.py $CONTENT_VERSION $GIT_SHA1 $CIRCLE_BUILD_NUM $SERVER_VERSION || echo "ignore errors"
+                # new release notes summary generator in packs format
+                python3 Utils/release_notes_generator.py $CONTENT_VERSION $GIT_SHA1 $CIRCLE_BUILD_NUM --output $CIRCLE_ARTIFACTS/packs-release-notes.md
             fi
       - run:
           name: Common Server Documentation
@@ -309,25 +323,11 @@ jobs:
                 echo 'Skipping - running only in container number 0.'
                 exit 0;
             fi
-            ./Tests/scripts/prepare_content_packs_for_testing.sh
-      - run:
-          name: Zip Content Packs From GCS
-          command: |
             if [[ $CIRCLE_BRANCH =~ pull/[0-9]+ ]]; then
               echo "Skipping, Should not run on contributor's branch."
               exit 0
             fi
-            if [ $CIRCLE_NODE_INDEX -ne 0 ] ;
-              then
-                echo 'Skipping - running only in container number 0.'
-                exit 0;
-            fi
-            GCS_PATH=$(mktemp)
-            ZIP_FOLDER=$(mktemp -d)
-            echo $GCS_MARKET_KEY > $GCS_PATH
-            python3 ./Tests/Marketplace/zip_packs.py -b 'marketplace-ci-build' -z $ZIP_FOLDER -a $CIRCLE_ARTIFACTS -s $GCS_PATH -n $CIRCLE_BUILD_NUM -br $CIRCLE_BRANCH
-            rm $GCS_PATH
-          when: always
+            ./Tests/scripts/prepare_content_packs_for_testing.sh
       - store_artifacts:
           path: artifacts
           destination: artifacts
@@ -352,7 +352,7 @@ jobs:
                     export IFRA_ENV_TYPE=Nightly # disable-secrets-detection
                 elif [ -n "${INSTANCE_TESTS}" ] ;
                   then
-                    export IFRA_ENV_TYPE="Demisto Marketplace" # disable-secrets-detection
+                    export IFRA_ENV_TYPE="Demisto PreGA" # disable-secrets-detection
 
                   else
                     export IFRA_ENV_TYPE=Content-Env # disable-secrets-detection
@@ -476,9 +476,9 @@ jobs:
               echo "Skipping instance tests for forked PRs"
               exit 0
             fi
-            if [ -n "${INSTANCE_TESTS}" ] || [ $CIRCLE_NODE_INDEX -ne 0 ] ;
+            if [ -n "${INSTANCE_TESTS}" ] || [ $CIRCLE_NODE_INDEX -ne 0 ] || [ -n "${NIGHTLY}" ];
               then
-                echo "Skipping - not running in INSTANCE_TESTS build or unit-tests container"
+                echo "Skipping - not running in INSTANCE_TESTS build, unit-tests container or Nightly run"
                 exit 0
             fi
             export TEMP=$(cat ./Tests/filter_envs.json | jq '."Demisto PreGA"')
@@ -508,9 +508,9 @@ jobs:
               echo "Skipping instance tests for forked PRs"
               exit 0
             fi
-            if [ -n "${INSTANCE_TESTS}" ] || [ $CIRCLE_NODE_INDEX -ne 0 ] || [ -n "${NIGHTLY}" ];
+            if [ -n "${INSTANCE_TESTS}" ] || [ $CIRCLE_NODE_INDEX -ne 0 ] ;
               then
-                echo "Skipping - not running in INSTANCE_TESTS build, unit-tests container or Nightly run"
+                echo "Skipping - not running in INSTANCE_TESTS build or unit-tests container"
                 exit 0
             fi
             export TEMP=$(cat ./Tests/filter_envs.json | jq '."Demisto Marketplace"')
@@ -556,6 +556,9 @@ jobs:
             if [ -n "${INSTANCE_TESTS}" ] && [ $CIRCLE_NODE_INDEX -ne 1 ] ;
               then
                 ./Tests/scripts/instance_test.sh
+                export RETVAL=$?
+                cp ./Tests/failed_instances.txt $CIRCLE_ARTIFACTS/failed_instances.txt
+                exit $RETVAL
               else
                 echo "Skipping instance tests"
                 exit 0
@@ -599,9 +602,28 @@ jobs:
             ID_SET=$CIRCLE_ARTIFACTS/id_set.json
             GCS_PATH=$(mktemp)
             echo $GCS_MARKET_KEY > $GCS_PATH
-            python3 ./Tests/Marketplace/upload_packs.py -a $PACK_ARTIFACTS -e $EXTRACT_FOLDER -b 'marketplace-dist' -d $PACKS_DEPENDENCIES -i $ID_SET -s $GCS_PATH -n $CIRCLE_BUILD_NUM -k $PACK_SIGNING_KEY -pb 'marketplace-dist-private' -o
+            python3 ./Tests/Marketplace/upload_packs.py -a $PACK_ARTIFACTS -e $EXTRACT_FOLDER -b 'marketplace-dist' -d $PACKS_DEPENDENCIES -i $ID_SET -s $GCS_PATH -n $CIRCLE_BUILD_NUM -k $PACK_SIGNING_KEY -pb 'marketplace-dist-private'
             rm $GCS_PATH
           when: on_success
+      - run:
+          name: Zip Content Packs From GCS
+          command: |
+            if [[ $CIRCLE_BRANCH != master ]]; then
+              echo "Skipping packs zipping on non master branch"
+              exit 0
+            fi
+
+            if [ $CIRCLE_NODE_INDEX -ne 0 ] ;
+              then
+                echo 'Skipping - running only in container number 0.'
+                exit 0;
+            fi
+            GCS_PATH=$(mktemp)
+            ZIP_FOLDER=$(mktemp -d)
+            echo $GCS_MARKET_KEY > $GCS_PATH
+            python3 ./Tests/Marketplace/zip_packs.py -b 'marketplace-dist' -z $ZIP_FOLDER -a $CIRCLE_ARTIFACTS -s $GCS_PATH -gp content/packs -rt False
+            rm $GCS_PATH
+          when: always
       - store_artifacts:
           path: artifacts
           destination: artifacts