prepare repo for auto-formatters

.circleci/config.yml.in

@@ -87,10 +87,62 @@ jobs:
     steps:
       - checkout
       - run:
+          name: Install check utilities
+          command: pip install --user --progress-bar=off jinja2 pyyaml
+      - run:
+          name: Check CircleCI config consistency
+          command: python .circleci/regenerate.py && git diff --quiet
+      - run:
+          when: on_fail
+          name: .circleci/config.yml not in sync with config.yml.in! Run '$ python .circleci/regenerate.py' to fix this.
+          command: exit 1
+
+  lint_python_and_config:
+    docker:
+      - image: circleci/python:3.7
+    steps:
+      - checkout
+      - run:
+          name: Install lint utilities
+          command: pip install --user --progress-bar=off pre-commit
+      - run:
+          name: Install pre-commit hooks
+          command: pre-commit install-hooks
+      - run:
+          name: Lint Python code and config files
+          command: pre-commit run --all-files
+      - run:
+          when: on_fail
+          name: Code format not compliant with the rules! Run '$ pre-commit run --all-files' to fix this.
+          command: exit 1
+
+  lint_c:
+    docker:
+      - image: circleci/python:3.7
+    steps:
+      - run:
+          name: Install additional system libraries
+          command: |
+            sudo apt update -qy
+            sudo apt install libtinfo5
+      - checkout
+      - run:
+          name: Install lint utilities
           command: |
-            pip install --user --progress-bar off jinja2 pyyaml
-            python .circleci/regenerate.py
-            git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)
+            curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o clang-format
+            chmod +x clang-format
+            ./clang-format --version
+      - run:
+          name: Lint C code
+          command: >
+            python run-clang-format.py
+            --recursive
+            --clang-format-executable=./clang-format
+            torchtext/csrc
+      - run:
+          when: on_fail
+          name: Code format not compliant with the rules! Run '$ python run-clang-format.py' to fix this.
+          command: exit 1
 
   binary_linux_wheel:
     <<: *binary_common
@@ -591,28 +643,14 @@ jobs:
             target=${tag:-main}
             ~/workspace/.circleci/build_docs/commit_docs.sh ~/workspace $target
 
-  docstring_parameters_sync:
-    <<: *binary_common
-    docker:
-      - image: continuumio/miniconda3
-    resource_class: medium
-    steps:
-      - attach_workspace:
-          at: ~/workspace
-      - designate_upload_channel
-      - checkout
-      - run:
-          name: Check parameters docstring sync
-          command: |
-            pip install --user pydocstyle
-            export PATH="$HOME/.local/bin:$PATH"
-            pydocstyle torchtext
-
-
 workflows:
-  build:
+  lint:
     jobs:
       - circleci_consistency
+      - lint_python_and_config
+      - lint_c
+  build:
+    jobs:
       {{ build_workflows() }}
   unittest:
     jobs:

.circleci/regenerate.py

@@ -42,7 +42,6 @@ def build_workflows(prefix='', upload=False, filter_branch=None, indentation=6):
         # Build on every pull request, but upload only on nightly and tags
         w += build_doc_job('/.*/')
         w += upload_doc_job('nightly')
-        w += docstring_parameters_sync_job(None)
 
     return indent(indentation, w)
 
@@ -85,18 +84,6 @@ def upload_doc_job(filter_branch):
     return [{"upload_docs": job}]
 
 
-def docstring_parameters_sync_job(filter_branch):
-    job = {
-        "name": "docstring_parameters_sync",
-        "python_version": "3.8",
-        "requires": ["binary_linux_wheel_py3.8", ],
-    }
-
-    if filter_branch:
-        job["filters"] = gen_filter_branch_tree(filter_branch)
-    return [{"docstring_parameters_sync": job}]
-
-
 def generate_base_workflow(base_workflow_name, python_version, filter_branch, os_type, btype):
     d = {
         "name": base_workflow_name,
@@ -167,7 +154,7 @@ def unittest_workflows(indentation=6):
             }
         })
 
-        for i, python_version in enumerate(PYTHON_VERSIONS):
+        for python_version in PYTHON_VERSIONS:
             w.append({
                 f"unittest_{os_type}": {
                     "name": f"unittest_{os_type}_py{python_version}",
@@ -176,13 +163,6 @@ def unittest_workflows(indentation=6):
                 }
             })
 
-            if i == 0 and os_type == "linux":
-                w.append({
-                    "stylecheck": {
-                        "name": f"stylecheck_py{python_version}",
-                        "python_version": python_version,
-                    }
-                })
     return indent(indentation, w)
 
 

.circleci/unittest/linux/scripts/setup_env.sh

@@ -37,11 +37,6 @@ conda activate "${env_dir}"
 # 3. Install Conda dependencies
 printf "* Installing dependencies (except PyTorch)\n"
 conda env update --file "${this_dir}/environment.yml" --prune
-if [ "${os}" == Linux ] ; then
-    clangformat_path="${root_dir}/clang-format"
-    curl https://oss-clang-format.s3.us-east-2.amazonaws.com/linux64/clang-format-linux64 -o "${clangformat_path}"
-    chmod +x "${clangformat_path}"
-fi
 
 # 4. Download
 printf "* Downloading SpaCy English models\n"

.flake8

@@ -1,5 +1,5 @@
 [flake8]
-# E501 is not flexible enough, we're using B950 instead. Consistent with pytorch
-ignore = E402,E722,W503,W504,F821,E501
+ignore = E401,E402,E501,E722,W503,W504,F821,B006,B007,B008,B009
+select = D417 # Missing argument descriptions in the docstring
 max-line-length = 120
 exclude = docs/source,third_party

.pre-commit-config.yaml

@@ -0,0 +1,35 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.0.1
+    hooks:
+      - id: trailing-whitespace
+      - id: mixed-line-ending
+        args:
+          - --fix=lf
+      - id: end-of-file-fixer
+
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v2.5.1
+    hooks:
+      - id: prettier
+        types_or:
+          - markdown
+          - toml
+          - yaml
+
+  - repo: https://github.com/omnilib/ufmt
+    rev: v1.3.1
+    hooks:
+      - id: ufmt
+        additional_dependencies:
+          - black == 21.12b0
+          - usort == 1.0.1
+
+  - repo: https://github.com/pycqa/flake8
+    rev: 4.0.1
+    hooks:
+      - id: flake8
+        additional_dependencies:
+          - flake8-docstrings == 1.6.0
+        args:
+          - --config=.flake8

.prettierignore

@@ -0,0 +1,2 @@
+packaging/*
+.circleci/config.yml

.prettierrc.yaml

@@ -0,0 +1,2 @@
+proseWrap: always
+printWidth: 120

CONTRIBUTING.md

@@ -9,8 +9,24 @@ We actively welcome your pull requests.
 2. If you've added code that should be tested, add tests.
 3. If you've changed APIs, update the documentation.
 4. Ensure the test suite passes.
-5. Make sure your code lints.
-6. If you haven't already, complete the Contributor License Agreement ("CLA").
+5. If you haven't already, complete the Contributor License Agreement ("CLA").
+
+### Code style
+
+`torchtext` enforces a fairly strict code format through [`pre-commit`](https://pre-commit.com). You can install it with
+
+```shell
+pip install pre-commit
+```
+
+or
+
+```shell
+conda install -c conda-forge pre-commit
+```
+
+To check and in most cases fix the code format, stage all your changes (`git add`) and execute `pre-commit run`. To perform
+the checks automatically before every `git commit`, you can install the checks as hooks with `pre-commit install`.
 
 ## Contributor License Agreement ("CLA")
 In order to accept your pull request, we need you to submit a CLA. You only need