Merge pull request #1814 from magento-pangolin/sprint-2.2-develop

[Pangolin] Deliver changes from Sprint 14 - 2.2-develop

 - MQE-528: Robo commands should override certain files.
 - MQE-382: Adding custom "assertElementContainsAttribute" method.
 - MQE-572: Expose selenium configuration from within our .env file
 - MQE-513: added attachment (screenshots/logs) support in Allure report
 - MQE-439: .env.example parameter example comment block
 - MQE-541: changed to use stepKey for test step sequencing.
 - MQE-569: added SampleTests in module blacklist and not allowed to generate
 - MQE-570: Sanitize MAGENTO_BASE_URL (.env file) and Selenium params
 - MQE-574: selector or selectorArray attribute is required for 'click'
 - MQE-583: set release version to 1.0.0 and updated composer dependencies.
 - MQE-347: Use a remove tag instead of a remove attribute
 - MQE-386: MFTF Compatibility with Windows Machine
 - MQE-592: Audit Test Cases on Firefox/Chrome on Windows/OSX
 - MQE-599: Move composer.json, README, etc from EE to CE

Author: okolesnyk

dev/tests/acceptance/.env.example

@@ -1,22 +1,55 @@
 #Copyright © Magento, Inc. All rights reserved.
 #See COPYING.txt for license details.
 
+#*** Start of example .env ***#
+#
+# MAGENTO_BASE_URL=http://127.0.0.1:32772/
+#
+# MAGENTO_BACKEND_NAME=admin
+# MAGENTO_ADMIN_USERNAME=admin
+# MAGENTO_ADMIN_PASSWORD=123123q
+#
+# SELENIUM_HOST=127.0.0.1
+# SELENIUM_PORT=4444
+# SELENIUM_PROTOCOL=http
+# SELENIUM_PATH=/wd/hub
+#
+# MAGENTO_RESTAPI_SERVER_HOST=127.0.0.1
+# MAGENTO_RESTAPI_SERVER_PORT=32769
+#
+# TESTS_BP=/Users/First_Last/GitHub/magento2ce/dev/tests/acceptance/tests/functional
+# FW_BP=/Users/First_Last/GitHub/magento2-functional-testing-framework
+# TESTS_MODULE_PATH=/Users/First_Last/GitHub/magento2ce/dev/tests/acceptance/tests/functional/Magento/FunctionalTest
+# MODULE_WHITELIST=Magento_NewModule
+#
+#*** End of example .env ***#
+
+
+#*** Start of .env ***#
+
+#*** Set the base URL for your Magento instance ***#
 MAGENTO_BASE_URL=
 
+#*** Set the Admin Username and Password for your Magento instance ***#
 MAGENTO_BACKEND_NAME=
 MAGENTO_ADMIN_USERNAME=
 MAGENTO_ADMIN_PASSWORD=
 
-#*** Uncomment and set host & port if your dev environment needs different value other than MAGENTO_BASE_URL for Rest Api Requests ***#
+#*** Selenium Server Protocol, Host, Port, and Path, with local defaults. Uncomment and change if not running Selenium locally.
+#SELENIUM_HOST=127.0.0.1
+#SELENIUM_PORT=4444
+#SELENIUM_PROTOCOL=http
+#SELENIUM_PATH=/wd/hub
+
+#*** Uncomment and set host & port if your dev environment needs different value other than MAGENTO_BASE_URL for Rest API Requests ***#
 #MAGENTO_RESTAPI_SERVER_HOST=
 #MAGENTO_RESTAPI_SERVER_PORT=
 
-DB_DSN=
-DB_USERNAME=
-DB_PASSWORD=
-
-#*** uncomment these properties to set up a dev environment with symlinked projects***#
+#*** Uncomment these properties to set up a dev environment with symlinked projects ***#
 #TESTS_BP=
 #FW_BP=
 #TESTS_MODULE_PATH=
-#MODULE_WHITELIST=
+#MODULE_WHITELIST=
+#MFTF_DEBUG=true
+
+#*** End of .env ***#

dev/tests/acceptance/README.md

@@ -0,0 +1,64 @@
+# Magento Functional Testing Framework
+
+----
+
+## System Requirements
+[Magento Functional Testing Framework system requirements](http://devdocs.magento.com/guides/v2.2/magento-functional-testing-framework/getting-started.html#prepare-environment)
+
+## Installation
+To install the Magento Functional Testing Framework, see [Getting Started](http://devdocs.magento.com/guides/v2.2/magento-functional-testing-framework/getting-started.html)
+
+## Contributing
+Contributions can take the form of new components or features, changes to existing features, tests, documentation (such as developer guides, user guides, examples, or specifications), bug fixes, optimizations, or just good suggestions.
+
+To learn about how to make a contribution, click [here][1].
+
+To open an issue, click [here][2].
+
+To suggest documentation improvements, click [here][3].
+
+[1]: <http://devdocs.magento.com/guides/v2.2/magento-functional-testing-framework/contribution-guidelines.html>
+[2]: <https://github.com/magento/magento2-functional-testing-framework/issues>
+[3]: <http://devdocs.magento.com>
+
+### Labels applied by the MFTF team
+
+Refer to the tables with descriptions of each label below. These labels are applied by the MFTF development team to community contributed issues and pull requests, to communicate status, impact, or which team is working on it.
+
+### Pull Request Status
+
+Label| Description
+---|---
+**accept**| The pull request has been accepted and will be merged into mainline code. 
+**reject**| The pull request has been rejected and will not be merged into mainline code. Possible reasons can include but are not limited to: issue has already been fixed in another code contribution, or there is an issue with the code contribution.
+**needsUpdate**| The Magento Team needs additional information from the reporter to properly prioritize and process the pull request.
+
+### Issue Resolution Status
+
+Label| Description
+---|---
+**acknowledged**| The Magento Team has validated the issue and an internal ticket has been created.
+**needsUpdate**| The Magento Team needs additional information from the reporter to properly prioritize and process the issue or pull request.
+**cannot reproduce**| The Magento Team has not confirmed that this issue contains the minimum required information to reproduce. 
+**non-issue**| The Magento Team has not recognised any issue according to provided information.
+
+### Domains Impacted
+
+Label| Description
+---|---
+**PROD**| Affects the Product team (mostly feature requests or business logic change).
+**DOC**| Affects Documentation domain.
+**TECH**| Affects Architect Group (mostly to make decisions around technology changes).
+
+### Type
+
+Label| Description
+---|---
+**bugfix**| The issue or pull request relates to bug fixing.
+**enhancement**| The issue or pull request that raises the MFTF to a higher degree (for example new features, optimization, refactoring, etc).
+
+## License
+
+Each Magento source file included in this distribution is licensed under APL 3.0
+
+Please see LICENSE_APL3.txt for the full text of the APL 3.0 license or contact [email protected] for a copy.

dev/tests/acceptance/RoboFile.php

@@ -21,8 +21,8 @@ class RoboFile extends \Robo\Tasks
     function cloneFiles()
     {
         $this->_exec('cp -vn .env.example .env');
-        $this->_exec('cp -vn codeception.dist.yml codeception.yml');
-        $this->_exec('cp -vn tests/functional.suite.dist.yml tests/functional.suite.yml');
+        $this->_exec('cp -vf codeception.dist.yml codeception.yml');
+        $this->_exec('cp -vf tests'. DIRECTORY_SEPARATOR .'functional.suite.dist.yml tests'. DIRECTORY_SEPARATOR .'functional.suite.yml');
     }
 
     /**
@@ -34,7 +34,7 @@ function cloneFiles()
     function buildProject()
     {
         $this->cloneFiles();
-        $this->_exec('./vendor/bin/codecept build');
+        $this->_exec('vendor'. DIRECTORY_SEPARATOR .'bin'. DIRECTORY_SEPARATOR .'codecept build');
     }
 
     /**
@@ -72,75 +72,45 @@ function generateSuite(array $args)
     }
 
     /**
-     * Run all Functional tests using the Chrome environment.
+     * Run all Functional tests.
      *
      * @return void
      */
-    function chrome()
+    function functional()
     {
-        $this->_exec('./vendor/bin/codecept run functional --env chrome --skip-group skip');
+        $this->_exec('.' . DIRECTORY_SEPARATOR . 'vendor' . DIRECTORY_SEPARATOR . 'bin' . DIRECTORY_SEPARATOR . 'codecept run functional --skip-group skip');
     }
 
     /**
-     * Run all Functional tests using the FireFox environment.
-     *
-     * @return void
-     */
-    function firefox()
-    {
-        $this->_exec('./vendor/bin/codecept run functional --env firefox --skip-group skip');
-    }
-
-    /**
-     * Run all Functional tests using the PhantomJS environment.
-     *
-     * @return void
-     */
-    function phantomjs()
-    {
-        $this->_exec('./vendor/bin/codecept run functional --env phantomjs --skip-group skip');
-    }
-
-    /**
-     * Run all Functional tests using the Chrome Headless environment.
-     *
-     * @return void
-     */
-    function headless()
-    {
-        $this->_exec('./vendor/bin/codecept run functional --env headless --skip-group skip');
-    }
-
-    /**
-     * Run all Tests with the specified @group tag, excluding @group 'skip', using the Chrome environment.
+     * Run all Tests with the specified @group tag, excluding @group 'skip'.
      *
      * @param string $args
      * @return void
      */
     function group($args = '')
     {
-        $this->taskExec('./vendor/bin/codecept run functional --verbose --steps --env chrome --skip-group skip --group')->args($args)->run();
+        $this->taskExec('.' . DIRECTORY_SEPARATOR . 'vendor' . DIRECTORY_SEPARATOR . 'bin' . DIRECTORY_SEPARATOR . 'codecept run functional --verbose --steps --skip-group skip --group')->args($args)->run();
     }
 
     /**
-     * Run all Functional tests located under the Directory Path provided using the Chrome environment.
+     * Run all Functional tests located under the Directory Path provided.
      *
      * @param string $args
      * @return void
      */
     function folder($args = '')
     {
-        $this->taskExec('./vendor/bin/codecept run functional --env chrome')->args($args)->run();
+        $this->taskExec('.' . DIRECTORY_SEPARATOR . 'vendor' . DIRECTORY_SEPARATOR . 'bin' . DIRECTORY_SEPARATOR . 'codecept run functional')->args($args)->run();
     }
 
     /**
-     * Run all Tests marked with the @group tag 'example', using the Chrome environment.
+     * Run all Tests marked with the @group tag 'example'.
      *
      * @return void
      */
     function example()
     {
-        $this->_exec('./vendor/bin/codecept run --env chrome --group example --skip-group skip');
+        $this->_exec('.' . DIRECTORY_SEPARATOR . 'vendor' . DIRECTORY_SEPARATOR . 'bin' . DIRECTORY_SEPARATOR . 'codecept run --group example --skip-group skip');
     }
 
     /**
@@ -150,7 +120,7 @@ function example()
      */
     function allure1Generate()
     {
-        return $this->_exec('allure generate tests/_output/allure-results/ -o tests/_output/allure-report/');
+        return $this->_exec('allure generate tests'. DIRECTORY_SEPARATOR .'_output'. DIRECTORY_SEPARATOR .'allure-results'. DIRECTORY_SEPARATOR .' -o tests'. DIRECTORY_SEPARATOR .'_output'. DIRECTORY_SEPARATOR .'allure-report'. DIRECTORY_SEPARATOR .'');
     }
 
     /**
@@ -160,7 +130,7 @@ function allure1Generate()
      */
     function allure2Generate()
     {
-        return $this->_exec('allure generate tests/_output/allure-results/ --output tests/_output/allure-report/ --clean');
+        return $this->_exec('allure generate tests'. DIRECTORY_SEPARATOR .'_output'. DIRECTORY_SEPARATOR .'allure-results'. DIRECTORY_SEPARATOR .' --output tests'. DIRECTORY_SEPARATOR .'_output'. DIRECTORY_SEPARATOR .'allure-report'. DIRECTORY_SEPARATOR .' --clean');
     }
 
     /**
@@ -170,7 +140,7 @@ function allure2Generate()
      */
     function allure1Open()
     {
-        $this->_exec('allure report open --report-dir tests/_output/allure-report/');
+        $this->_exec('allure report open --report-dir tests'. DIRECTORY_SEPARATOR .'_output'. DIRECTORY_SEPARATOR .'allure-report'. DIRECTORY_SEPARATOR .'');
     }
 
     /**
@@ -180,7 +150,7 @@ function allure1Open()
      */
     function allure2Open()
     {
-        $this->_exec('allure open --port 0 tests/_output/allure-report/');
+        $this->_exec('allure open --port 0 tests'. DIRECTORY_SEPARATOR .'_output'. DIRECTORY_SEPARATOR .'allure-report'. DIRECTORY_SEPARATOR .'');
     }
 
     /**

dev/tests/acceptance/composer.json

@@ -0,0 +1,35 @@
+{
+    "name": "magento/magento2ce-functional-tests",
+    "description": "Magento 2 (Open Source) Functional Tests",
+    "type": "project",
+    "version": "1.0.0-dev",
+    "license": [
+        "OSL-3.0",
+        "AFL-3.0"
+    ],
+    "config": {
+        "sort-packages": true
+    },
+    "repositories": [
+        {
+            "type": "git",
+            "url": "[email protected]:magento/magento2-functional-testing-framework.git"
+        }
+    ],
+    "require": {
+        "allure-framework/allure-codeception": "dev-master#af40af5ae2b717618a42fe3e137d75878508c75d",
+        "codeception/codeception": "~2.3.4",
+        "consolidation/robo": "^1.0.0",
+        "symfony/process": ">=2.7 <3.4",
+        "henrikbjorn/lurker": "^1.2",
+        "magento/magento2-functional-testing-framework": "1.0.0",
+        "php": "7.0.2|7.0.4|~7.0.6|~7.1.0",
+        "vlucas/phpdotenv": "~2.4"
+    },
+    "autoload": {
+        "psr-4": {
+            "Magento\\": "tests/functional/Magento"
+        }
+    },
+    "prefer-stable": true
+}

dev/tests/acceptance/tests/_bootstrap.php

@@ -22,3 +22,9 @@
     }
 }
 defined('FW_BP') || define('FW_BP', PROJECT_ROOT . $RELATIVE_FW_PATH);
+
+// add the debug flag here
+$debug_mode = $_ENV['MFTF_DEBUG'] ?? false;
+if (!(bool)$debug_mode && extension_loaded('xdebug')) {
+    xdebug_disable();
+}

dev/tests/acceptance/tests/_suite/sampleSuite.xml

@@ -9,14 +9,14 @@
 <suites xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../../vendor/magento/magento2-functional-testing-framework/src/Magento/FunctionalTestingFramework/Suite/etc/suiteSchema.xsd">
     <suite name="mySuite">
         <before>
-            <createData entity="_defaultCategory" mergeKey="createCategory"/>
-            <createData entity="_defaultProduct" mergeKey="createProduct">
+            <createData entity="_defaultCategory" stepKey="createCategory"/>
+            <createData entity="_defaultProduct" stepKey="createProduct">
                 <required-entity createDataKey="createCategory"/>
             </createData>
         </before>
         <after>
-            <deleteData mergeKey="deleteMyProduct" createDataKey="createProduct"/>
-            <deleteData mergeKey="deleteMyCategory" createDataKey="createCategory"/>
+            <deleteData stepKey="deleteMyProduct" createDataKey="createProduct"/>
+            <deleteData stepKey="deleteMyCategory" createDataKey="createCategory"/>
         </after>
         <include>
             <group name="example"/>

dev/tests/acceptance/tests/functional.suite.dist.yml

@@ -31,3 +31,11 @@ modules:
             username: "%MAGENTO_ADMIN_USERNAME%"
             password: "%MAGENTO_ADMIN_PASSWORD%"
             pageload_timeout: 30
+            host: %SELENIUM_HOST%
+            port: %SELENIUM_PORT%
+            protocol: %SELENIUM_PROTOCOL%
+            path: %SELENIUM_PATH%
+            capabilities:
+              chromeOptions:
+                args: ["--start-maximized", "--disable-extensions", "--enable-automation"]
+

dev/tests/acceptance/tests/functional/Magento/FunctionalTest/AdminNotification/README.md

@@ -1,3 +1,3 @@
-# Magento 2 Acceptance Tests
+# Magento 2 Functional Tests
 
-The Acceptance Tests Module for **Magento_AdminNotification** Module.
+The Functional Tests Module for **Magento_AdminNotification** Module.