feat: add incremental linting

Author: dshemetov

.github/workflows/ci.yaml

@@ -65,7 +65,7 @@ jobs:
           docker network create --driver bridge delphi-net
           docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata --cap-add=sys_nice delphi_database_epidata
           docker run --rm -d -p 6379:6379 --network delphi-net --env "REDIS_PASSWORD=1234" --name delphi_redis delphi_redis
-          
+
 
       - run: |
           wget https://raw.githubusercontent.com/eficode/wait-for/master/wait-for

.github/workflows/lint.yaml

@@ -0,0 +1,40 @@
+name: Lint
+
+on:
+  pull_request:
+    branches:
+      - dev
+
+jobs:
+    darker:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v3
+              with:
+                fetch-depth: 0
+            - uses: actions/setup-python@v4
+            - uses: akaihola/[email protected]
+              with:
+                options: "--check --diff --isort --color"
+                version: "~=1.7.2"
+
+    lint-diffs:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Commit Range
+              id: commit-range
+              uses: akaihola/darker/.github/actions/[email protected]
+            - uses: actions/checkout@v3
+              with:
+                fetch-depth: 0
+            - uses: actions/setup-python@v4
+              with:
+                python-version: '3.8'
+                cache: 'pip'
+            - name: Install dependencies
+              run: |
+                python -m pip install --upgrade pip
+                pip install -r requirements.lint.txt
+            - name: Lint
+              run: |
+                inv lint --diff --no-format --revision=${{ steps.commit-range.outputs.commit-range }}

.gitignore

@@ -11,3 +11,5 @@ __pycache__/
 /node_modules
 .mypy_cache
 /missing_db_signals.csv
+venv
+env

README.md

@@ -64,6 +64,20 @@ $ cd repos
 $ pip install -e . --config-settings editable_mode=strict
 ```
 
+## Linting and Formatting
+
+We use [darker](https://github.com/akaihola/darker) and
+[lint-diffs](https://github.com/AtakamaLLC/lint-diffs/) to incrementally bring
+this repo up to PEP8 compliance. There is CI to ensure that all new code is
+compliant. To run the linter locally:
+
+```sh
+# Install lint dependencies
+pip install -r requirements.lint.txt
+# Run lint
+inv lint
+```
+
 # COVIDcast
 
 At the present, our primary focus is developing and expanding the

pyproject.toml

@@ -1,11 +1,16 @@
 [tool.black]
-line-length = 100
+line-length = 120
 target-version = ['py38']
-include = 'server,tests/server'
+
+[tool.isort]
+profile = "black"
+known_third_party = ["pytest"]
 
 [tool.pylint]
+    # Settings copied from
+    # cmu-delphi/covidcast-indicators/_delphi_utils_python/.pylintrc
     [tool.pylint.'MESSAGES CONTROL']
-    max-line-length = 100
+    max-line-length = 120
     disable = [
         'logging-format-interpolation',
         # Allow pytest functions to be part of a class

requirements.lint.txt

@@ -0,0 +1,6 @@
+darker[flynt]~=1.7.2
+invoke>=1.4.1
+isort==5.12.0
+lint_diffs==0.1.22
+pylint==2.8.3
+requests

tasks.py

@@ -1,3 +1,8 @@
+"""Repo tasks."""
+import pathlib
+import sys
+
+import requests
 from invoke import task
 
 
@@ -12,9 +17,6 @@ def update_gdoc(
     sources_url="https://docs.google.com/spreadsheets/d/e/2PACX-1vRfXo-qePhrYGAoZqewVnS1kt9tfnUTLgtkV7a-1q7yg4FoZk0NNGuB1H6k10ah1Xz5B8l1S1RB17N6/pub?gid=0&single=true&output=csv",
     signal_url="https://docs.google.com/spreadsheets/d/e/2PACX-1vRfXo-qePhrYGAoZqewVnS1kt9tfnUTLgtkV7a-1q7yg4FoZk0NNGuB1H6k10ah1Xz5B8l1S1RB17N6/pub?gid=329338228&single=true&output=csv",
 ):
-    import requests
-    import pathlib
-
     base_dir = pathlib.Path("./src/server/endpoints/covidcast_utils/")
 
     def _migrate_file(url: str, filename: str):
@@ -26,3 +28,41 @@ def _migrate_file(url: str, filename: str):
 
     _migrate_file(sources_url, "db_sources.csv")
     _migrate_file(signal_url, "db_signals.csv")
+
+
+@task
+def lint(c, incremental=True, format=True, revision="origin/dev", diff=False): # pylint: disable=redefined-builtin
+    """Lint and format.
+
+    Additional linter settings can be found in `pyproject.toml` (this invocation
+    will use those settings as well).
+
+    Parameters
+    ----------
+    incremental : bool
+        Only lint changed files.
+    format : bool
+        Apply formatting changes.
+    revision : str
+        Revision to compare against.
+    diff : bool
+        Only show formatting changes, do not apply.
+    """
+    if not incremental:
+        reponse = input("This will format all files in this repo, continue? [y/N]")
+        if reponse.lower() not in ("y", "yes"):
+            return
+
+    diff = "--diff" if diff else ""
+    if incremental:
+        if format:
+            c.run(f"darker --revision {revision}... {diff} --flynt --isort --color --check .")
+        out = c.run(f"git diff -U0 {revision} | lint-diffs")
+        if out.stdout.strip() != "=== pylint: mine=0, always=0":
+            print(out.stdout)
+            sys.exit(1)
+    else:
+        if format:
+            c.run(f"black {diff} .")
+            c.run(f"isort {diff} .")
+        c.run("pylint src/ tests/ integrations/")