Configure mypy to run in virtual environment and fix resulting issues (#989)

For some time I've noticed that my IDE is reporting mypy errors that the
pre-commit hook is not picking up. This is because [mypy
mirror](https://github.com/pre-commit/mirrors-mypy) runs in an isolated
pre-commit environment which does not have dependencies installed and it
enables `--ignore-missing-imports` to avoid errors.

This is [advised against by
mypy](https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-library-stubs-or-py-typed-marker)
for obvious reasons:

> We recommend avoiding --ignore-missing-imports if possible: it’s
equivalent to adding a # type: ignore to all unresolved imports in your
codebase.

This PR configures the mypy pre-commit hook to run in the virtual
environment and addresses the additional errors identified as a result.
It also introduces a minimal mypy config into the `pyproject.toml`

[mypy errors identified without the fixes in this
PR](https://github.com/user-attachments/files/15896693/mypyerrors.txt)

.github/workflows/ci.yml

@@ -27,6 +27,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
           cache-pre-commit: true
           poetry-version: ${{ env.POETRY_VERSION }}
+          poetry-install-options: "--all-extras"
       - name: "Check supported device md files are up to date"
         run: |
           poetry run pre-commit run generate-supported --all-files

.pre-commit-config.yaml

@@ -16,17 +16,6 @@ repos:
       args: [--fix, --exit-non-zero-on-fix]
     - id: ruff-format
 
-- repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.9.0
-  hooks:
-  - id: mypy
-    additional_dependencies: [types-click]
-    exclude: |
-      (?x)^(
-          kasa/modulemapping\.py|
-      )$
-
-
 - repo: https://github.com/PyCQA/doc8
   rev: 'v1.1.1'
   hooks:
@@ -35,6 +24,18 @@ repos:
 
 - repo: local
   hooks:
+  # Run mypy in the virtual environment so it uses the installed dependencies
+  # for more accurate checking than using the pre-commit mypy mirror
+  - id: mypy
+    name: mypy
+    entry: devtools/run-in-env.sh mypy
+    language: script
+    types_or: [python, pyi]
+    require_serial: true
+    exclude: |  # exclude required because --all-files passes py and pyi
+      (?x)^(
+          kasa/modulemapping\.py|
+      )$
   - id: generate-supported
     name: Generate supported devices
     description: This hook generates the supported device sections of README.md and SUPPORTED.md

devtools/bench/benchmark.py

@@ -5,8 +5,9 @@ import timeit
 
 import orjson
 from kasa_crypt import decrypt, encrypt
-from utils.data import REQUEST, WIRE_RESPONSE
+
-from utils.original import OriginalTPLinkSmartHomeProtocol
+from devtools.bench.utils.data import REQUEST, WIRE_RESPONSE
+from devtools.bench.utils.original import OriginalTPLinkSmartHomeProtocol
 
 
 def original_request_response() -> None:

devtools/run-in-env.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+source $(poetry env info --path)/bin/activate
+exec "$@"

kasa/aestransport.py

@@ -371,7 +371,10 @@ class AesEncyptionSession:
         handshake_key_bytes: bytes = base64.b64decode(handshake_key.encode("UTF-8"))
         private_key_data = base64.b64decode(keypair.get_private_key().encode("UTF-8"))
 
-        private_key = serialization.load_der_private_key(private_key_data, None, None)
+        private_key = cast(
+            rsa.RSAPrivateKey,
+            serialization.load_der_private_key(private_key_data, None, None),
+        )
         key_and_iv = private_key.decrypt(
             handshake_key_bytes, asymmetric_padding.PKCS1v15()
         )

kasa/cli.py

@@ -101,9 +101,7 @@ DEVICE_FAMILY_TYPES = [device_family_type.value for device_family_type in Device
 # Block list of commands which require no update
 SKIP_UPDATE_COMMANDS = ["raw-command", "command"]
 
-click.anyio_backend = "asyncio"
+pass_dev = click.make_pass_decorator(Device)  # type: ignore[type-abstract]
-
-pass_dev = click.make_pass_decorator(Device)
 
 
 def CatchAllExceptions(cls):
@@ -1005,7 +1003,7 @@ async def time_get(dev: Device):
 
 @time.command(name="sync")
 @pass_dev
-async def time_sync(dev: SmartDevice):
+async def time_sync(dev: Device):
     """Set the device time to current time."""
     if not isinstance(dev, SmartDevice):
         raise NotImplementedError("setting time currently only implemented on smart")
@@ -1143,7 +1141,7 @@ async def presets(ctx):
 
 @presets.command(name="list")
 @pass_dev
-def presets_list(dev: IotBulb):
+def presets_list(dev: Device):
     """List presets."""
     if not dev.is_bulb or not isinstance(dev, IotBulb):
         error("Presets only supported on iot bulbs")
@@ -1162,7 +1160,7 @@ def presets_list(dev: IotBulb):
 @click.option("--saturation", type=int)
 @click.option("--temperature", type=int)
 @pass_dev
-async def presets_modify(dev: IotBulb, index, brightness, hue, saturation, temperature):
+async def presets_modify(dev: Device, index, brightness, hue, saturation, temperature):
     """Modify a preset."""
     for preset in dev.presets:
         if preset.index == index:
@@ -1190,7 +1188,7 @@ async def presets_modify(dev: IotBulb, index, brightness, hue, saturation, tempe
 @click.option("--type", type=click.Choice(["soft", "hard"], case_sensitive=False))
 @click.option("--last", is_flag=True)
 @click.option("--preset", type=int)
-async def turn_on_behavior(dev: IotBulb, type, last, preset):
+async def turn_on_behavior(dev: Device, type, last, preset):
     """Modify bulb turn-on behavior."""
     if not dev.is_bulb or not isinstance(dev, IotBulb):
         error("Presets only supported on iot bulbs")
@@ -1248,7 +1246,7 @@ async def shell(dev: Device):
     logging.getLogger("asyncio").setLevel(logging.WARNING)
     loop = asyncio.get_event_loop()
     try:
-        await embed(
+        await embed(  # type: ignore[func-returns-value]
             globals=globals(),
             locals=locals(),
             return_asyncio_coroutine=True,

kasa/httpclient.py

@@ -36,7 +36,7 @@ class HttpClient:
 
     def __init__(self, config: DeviceConfig) -> None:
         self._config = config
-        self._client_session: aiohttp.ClientSession = None
+        self._client_session: aiohttp.ClientSession | None = None
         self._jar = aiohttp.CookieJar(unsafe=True, quote_cookie=False)
         self._last_url = URL(f"http://{self._config.host}/")
 

kasa/tests/discovery_fixtures.py

@@ -231,7 +231,9 @@ def discovery_data(request, mocker):
         return {"system": {"get_sysinfo": fixture_data["system"]["get_sysinfo"]}}
 
 
-@pytest.fixture(params=UNSUPPORTED_DEVICES.values(), ids=UNSUPPORTED_DEVICES.keys())
+@pytest.fixture(
+    params=UNSUPPORTED_DEVICES.values(), ids=list(UNSUPPORTED_DEVICES.keys())
+)
 def unsupported_device_info(request, mocker):
     """Return unsupported devices for cli and discovery tests."""
     discovery_data = request.param

kasa/tests/fakeprotocol_smart.py

@@ -276,7 +276,7 @@ class FakeSmartTransport(BaseTransport):
                     ):
                         result["sum"] = len(result[list_key])
                         if self.warn_fixture_missing_methods:
-                            pytest.fixtures_missing_methods.setdefault(
+                            pytest.fixtures_missing_methods.setdefault(  # type: ignore[attr-defined]
                                 self.fixture_name, set()
                             ).add(f"{method} (incomplete '{list_key}' list)")
 
@@ -305,7 +305,7 @@ class FakeSmartTransport(BaseTransport):
                 }
             # Reduce warning spam by consolidating and reporting at the end of the run
             if self.warn_fixture_missing_methods:
-                pytest.fixtures_missing_methods.setdefault(
+                pytest.fixtures_missing_methods.setdefault(  # type: ignore[attr-defined]
                     self.fixture_name, set()
                 ).add(method)
             return retval

kasa/tests/smart/modules/test_firmware.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import asyncio
 import logging
+from typing import TypedDict
 
 import pytest
 from pytest_mock import MockerFixture
@@ -71,7 +72,17 @@ async def test_firmware_update(
     assert fw
 
     upgrade_time = 5
-    extras = {"reboot_time": 5, "upgrade_time": upgrade_time, "auto_upgrade": False}
+
+    class Extras(TypedDict):
+        reboot_time: int
+        upgrade_time: int
+        auto_upgrade: bool
+
+    extras: Extras = {
+        "reboot_time": 5,
+        "upgrade_time": upgrade_time,
+        "auto_upgrade": False,
+    }
     update_states = [
         # Unknown 1
         DownloadState(status=1, download_progress=0, **extras),

kasa/tests/test_device.py

@@ -6,6 +6,7 @@ import importlib
 import inspect
 import pkgutil
 import sys
+from contextlib import AbstractContextManager
 from unittest.mock import Mock, patch
 
 import pytest
@@ -161,7 +162,7 @@ async def _test_attribute(
     dev: Device, attribute_name, is_expected, module_name, *args, will_raise=False
 ):
     if is_expected and will_raise:
-        ctx = pytest.raises(will_raise)
+        ctx: AbstractContextManager = pytest.raises(will_raise)
     elif is_expected:
         ctx = pytest.deprecated_call(match=(f"{attribute_name} is deprecated, use:"))
     else:

kasa/tests/test_emeter.py

@@ -5,7 +5,7 @@ import pytest
 from voluptuous import (
     All,
     Any,
-    Coerce,  # type: ignore
+    Coerce,
     Range,
     Schema,
 )
@@ -21,14 +21,14 @@ CURRENT_CONSUMPTION_SCHEMA = Schema(
     Any(
         {
             "voltage": Any(All(float, Range(min=0, max=300)), None),
-            "power": Any(Coerce(float, Range(min=0)), None),
+            "power": Any(Coerce(float), None),
-            "total": Any(Coerce(float, Range(min=0)), None),
+            "total": Any(Coerce(float), None),
-            "current": Any(All(float, Range(min=0)), None),
+            "current": Any(All(float), None),
             "voltage_mv": Any(All(float, Range(min=0, max=300000)), int, None),
-            "power_mw": Any(Coerce(float, Range(min=0)), None),
+            "power_mw": Any(Coerce(float), None),
-            "total_wh": Any(Coerce(float, Range(min=0)), None),
+            "total_wh": Any(Coerce(float), None),
-            "current_ma": Any(All(float, Range(min=0)), int, None),
+            "current_ma": Any(All(float), int, None),
-            "slot_id": Any(Coerce(int, Range(min=0)), None),
+            "slot_id": Any(Coerce(int), None),
         },
         None,
     )

kasa/tests/test_httpclient.py

@@ -38,7 +38,7 @@ from ..httpclient import HttpClient
         ),
         (Exception(), KasaException, "Unable to query the device: "),
         (
-            aiohttp.ServerFingerprintMismatch("exp", "got", "host", 1),
+            aiohttp.ServerFingerprintMismatch(b"exp", b"got", "host", 1),
             KasaException,
             "Unable to query the device: ",
         ),
@@ -84,7 +84,7 @@ async def test_httpclient_errors(mocker, error, error_raises, error_message, moc
     client = HttpClient(DeviceConfig(host))
     # Exceptions with parameters print with double quotes, without use single quotes
     full_msg = (
-        "\("  # type: ignore
+        re.escape("(")
         + "['\"]"
         + re.escape(f"{error_message}{host}: {error}")
         + "['\"]"

kasa/tests/test_iotdevice.py

@@ -207,7 +207,7 @@ async def test_mac(dev):
 
 @device_iot
 async def test_representation(dev):
-    pattern = re.compile("<DeviceType\..+ at .+? - .*? \(.+?\)>")
+    pattern = re.compile(r"<DeviceType\..+ at .+? - .*? \(.+?\)>")
     assert pattern.match(str(dev))
 
 

kasa/xortransport.py

@@ -229,7 +229,7 @@ class XorEncryption:
 try:
     from kasa_crypt import decrypt, encrypt
 
-    XorEncryption.decrypt = decrypt  # type: ignore[method-assign]
+    XorEncryption.decrypt = decrypt  # type: ignore[assignment]
-    XorEncryption.encrypt = encrypt  # type: ignore[method-assign]
+    XorEncryption.encrypt = encrypt  # type: ignore[assignment]
 except ImportError:
     pass

poetry.lock

@@ -1096,6 +1096,64 @@ files = [
     {file = "multidict-6.0.5.tar.gz", hash = "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da"},
 ]
 
+[[package]]
+name = "mypy"
+version = "1.9.0"
+description = "Optional static typing for Python"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "mypy-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f8a67616990062232ee4c3952f41c779afac41405806042a8126fe96e098419f"},
+    {file = "mypy-1.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d357423fa57a489e8c47b7c85dfb96698caba13d66e086b412298a1a0ea3b0ed"},
+    {file = "mypy-1.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49c87c15aed320de9b438ae7b00c1ac91cd393c1b854c2ce538e2a72d55df150"},
+    {file = "mypy-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:48533cdd345c3c2e5ef48ba3b0d3880b257b423e7995dada04248725c6f77374"},
+    {file = "mypy-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:4d3dbd346cfec7cb98e6cbb6e0f3c23618af826316188d587d1c1bc34f0ede03"},
+    {file = "mypy-1.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:653265f9a2784db65bfca694d1edd23093ce49740b2244cde583aeb134c008f3"},
+    {file = "mypy-1.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a3c007ff3ee90f69cf0a15cbcdf0995749569b86b6d2f327af01fd1b8aee9dc"},
+    {file = "mypy-1.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2418488264eb41f69cc64a69a745fad4a8f86649af4b1041a4c64ee61fc61129"},
+    {file = "mypy-1.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:68edad3dc7d70f2f17ae4c6c1b9471a56138ca22722487eebacfd1eb5321d612"},
+    {file = "mypy-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:85ca5fcc24f0b4aeedc1d02f93707bccc04733f21d41c88334c5482219b1ccb3"},
+    {file = "mypy-1.9.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aceb1db093b04db5cd390821464504111b8ec3e351eb85afd1433490163d60cd"},
+    {file = "mypy-1.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0235391f1c6f6ce487b23b9dbd1327b4ec33bb93934aa986efe8a9563d9349e6"},
+    {file = "mypy-1.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4d5ddc13421ba3e2e082a6c2d74c2ddb3979c39b582dacd53dd5d9431237185"},
+    {file = "mypy-1.9.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:190da1ee69b427d7efa8aa0d5e5ccd67a4fb04038c380237a0d96829cb157913"},
+    {file = "mypy-1.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:fe28657de3bfec596bbeef01cb219833ad9d38dd5393fc649f4b366840baefe6"},
+    {file = "mypy-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e54396d70be04b34f31d2edf3362c1edd023246c82f1730bbf8768c28db5361b"},
+    {file = "mypy-1.9.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5e6061f44f2313b94f920e91b204ec600982961e07a17e0f6cd83371cb23f5c2"},
+    {file = "mypy-1.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a10926e5473c5fc3da8abb04119a1f5811a236dc3a38d92015cb1e6ba4cb9e"},
+    {file = "mypy-1.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b685154e22e4e9199fc95f298661deea28aaede5ae16ccc8cbb1045e716b3e04"},
+    {file = "mypy-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:5d741d3fc7c4da608764073089e5f58ef6352bedc223ff58f2f038c2c4698a89"},
+    {file = "mypy-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:587ce887f75dd9700252a3abbc9c97bbe165a4a630597845c61279cf32dfbf02"},
+    {file = "mypy-1.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f88566144752999351725ac623471661c9d1cd8caa0134ff98cceeea181789f4"},
+    {file = "mypy-1.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61758fabd58ce4b0720ae1e2fea5cfd4431591d6d590b197775329264f86311d"},
+    {file = "mypy-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e49499be624dead83927e70c756970a0bc8240e9f769389cdf5714b0784ca6bf"},
+    {file = "mypy-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:571741dc4194b4f82d344b15e8837e8c5fcc462d66d076748142327626a1b6e9"},
+    {file = "mypy-1.9.0-py3-none-any.whl", hash = "sha256:a260627a570559181a9ea5de61ac6297aa5af202f06fd7ab093ce74e7181e43e"},
+    {file = "mypy-1.9.0.tar.gz", hash = "sha256:3cc5da0127e6a478cddd906068496a97a7618a21ce9b54bde5bf7e539c7af974"},
+]
+
+[package.dependencies]
+mypy-extensions = ">=1.0.0"
+tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""}
+typing-extensions = ">=4.1.0"
+
+[package.extras]
+dmypy = ["psutil (>=4.0)"]
+install-types = ["pip"]
+mypyc = ["setuptools (>=50)"]
+reports = ["lxml"]
+
+[[package]]
+name = "mypy-extensions"
+version = "1.0.0"
+description = "Type system extensions for programs checked with the mypy type checker."
+optional = false
+python-versions = ">=3.5"
+files = [
+    {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"},
+    {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"},
+]
+
 [[package]]
 name = "myst-parser"
 version = "1.0.0"
@@ -2185,4 +2243,4 @@ speedups = ["kasa-crypt", "orjson"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.8"
-content-hash = "f8edc1401028d0654bd4622bf471668dd84b323434f0fa40e783e5c9b45511f6"
+content-hash = "3aa0872e4188aad6e75025d47b026fa2a922bf039df38bfaac15f409e38d6889"

pyproject.toml

@@ -58,6 +58,7 @@ xdoctest = "*"
 coverage = {version = "*", extras = ["toml"]}
 pytest-timeout = "^2"
 pytest-freezer = "^0.4"
+mypy = "1.9.0"
 
 [tool.poetry.extras]
 docs = ["sphinx", "sphinx_rtd_theme", "sphinxcontrib-programoutput", "myst-parser", "docutils"]
@@ -138,3 +139,19 @@ convention = "pep257"
     "D100",
     "D103",
 ]
+
+[tool.mypy]
+warn_unused_configs = true  # warns if overrides sections unused/mis-spelled
+
+[[tool.mypy.overrides]]
+module = [ "kasa.tests.*", "devtools.*" ]
+disable_error_code = "annotation-unchecked"
+
+[[tool.mypy.overrides]]
+module = [
+    "devtools.bench.benchmark",
+    "devtools.parse_pcap",
+    "devtools.perftest",
+    "devtools.create_module_fixtures"
+]
+disable_error_code = "import-not-found,import-untyped"