Fix test framework running against real devices (#1235)

kasa/device.py

@@ -320,6 +320,11 @@ class Device(ABC):
     def model(self) -> str:
         """Returns the device model."""
 
+    @property
+    @abstractmethod
+    def _model_region(self) -> str:
+        """Return device full model name and region."""
+
     @property
     @abstractmethod
     def alias(self) -> str | None:

kasa/iot/iotdevice.py

@@ -455,6 +455,12 @@ class IotDevice(Device):
         sys_info = self._sys_info
         return str(sys_info["model"])
 
+    @property
+    @requires_update
+    def _model_region(self) -> str:
+        """Return device full model name and region."""
+        return self.model
+
     @property  # type: ignore
     def alias(self) -> str | None:
         """Return device name (alias)."""

kasa/smart/smartdevice.py

@@ -492,6 +492,17 @@ class SmartDevice(Device):
         """Returns the device model."""
         return str(self._info.get("model"))
 
+    @property
+    def _model_region(self) -> str:
+        """Return device full model name and region."""
+        if (disco := self._discovery_info) and (
+            disco_model := disco.get("device_model")
+        ):
+            return disco_model
+        # Some devices have the region in the specs element.
+        region = f"({specs})" if (specs := self._info.get("specs")) else ""
+        return f"{self.model}{region}"
+
     @property
     def alias(self) -> str | None:
         """Returns the device alias or nickname."""

tests/device_fixtures.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import os
 from collections.abc import AsyncGenerator
 
 import pytest
@@ -142,7 +143,7 @@ ALL_DEVICES_SMART = (
 )
 ALL_DEVICES = ALL_DEVICES_IOT.union(ALL_DEVICES_SMART)
 
-IP_MODEL_CACHE: dict[str, str] = {}
+IP_FIXTURE_CACHE: dict[str, FixtureInfo] = {}
 
 
 def parametrize_combine(parametrized: list[pytest.MarkDecorator]):
@@ -448,6 +449,39 @@ def get_fixture_info(fixture, protocol):
             return fixture_info
 
 
+def get_nearest_fixture_to_ip(dev):
+    if isinstance(dev, SmartDevice):
+        protocol_fixtures = filter_fixtures("", protocol_filter={"SMART"})
+    elif isinstance(dev, SmartCamera):
+        protocol_fixtures = filter_fixtures("", protocol_filter={"SMARTCAMERA"})
+    else:
+        protocol_fixtures = filter_fixtures("", protocol_filter={"IOT"})
+    assert protocol_fixtures, "Unknown device type"
+
+    # This will get the best fixture with a match on model region
+    if model_region_fixtures := filter_fixtures(
+        "", model_filter={dev._model_region}, fixture_list=protocol_fixtures
+    ):
+        return next(iter(model_region_fixtures))
+
+    # This will get the best fixture based on model starting with the name.
+    if "(" in dev.model:
+        model, _, _ = dev.model.partition("(")
+    else:
+        model = dev.model
+    if model_fixtures := filter_fixtures(
+        "", model_startswith_filter=model, fixture_list=protocol_fixtures
+    ):
+        return next(iter(model_fixtures))
+
+    if device_type_fixtures := filter_fixtures(
+        "", device_type_filter={dev.device_type}, fixture_list=protocol_fixtures
+    ):
+        return next(iter(device_type_fixtures))
+
+    return next(iter(protocol_fixtures))
+
+
 @pytest.fixture(params=filter_fixtures("main devices"), ids=idgenerator)
 async def dev(request) -> AsyncGenerator[Device, None]:
     """Device fixture.
@@ -459,24 +493,28 @@ async def dev(request) -> AsyncGenerator[Device, None]:
     dev: Device
 
     ip = request.config.getoption("--ip")
-    username = request.config.getoption("--username")
-    password = request.config.getoption("--password")
+    username = request.config.getoption("--username") or os.environ.get("KASA_USERNAME")
+    password = request.config.getoption("--password") or os.environ.get("KASA_PASSWORD")
     if ip:
-        model = IP_MODEL_CACHE.get(ip)
-        d = None
-        if not model:
-            d = await _discover_update_and_close(ip, username, password)
-            IP_MODEL_CACHE[ip] = model = d.model
+        fixture = IP_FIXTURE_CACHE.get(ip)
 
-        if model not in fixture_data.name:
+        d = None
+        if not fixture:
+            d = await _discover_update_and_close(ip, username, password)
+            IP_FIXTURE_CACHE[ip] = fixture = get_nearest_fixture_to_ip(d)
+        assert fixture
+        if fixture.name != fixture_data.name:
             pytest.skip(f"skipping file {fixture_data.name}")
-        dev = d if d else await _discover_update_and_close(ip, username, password)
+            dev = None
+        else:
+            dev = d if d else await _discover_update_and_close(ip, username, password)
     else:
         dev = await get_device_for_fixture(fixture_data)
 
     yield dev
 
-    await dev.disconnect()
+    if dev:
+        await dev.disconnect()
 
 
 def get_parent_and_child_modules(device: Device, module_name):

tests/fixtureinfo.py

@@ -104,8 +104,10 @@ def filter_fixtures(
     data_root_filter: str | None = None,
     protocol_filter: set[str] | None = None,
     model_filter: set[str] | None = None,
+    model_startswith_filter: str | None = None,
     component_filter: str | ComponentFilter | None = None,
     device_type_filter: Iterable[DeviceType] | None = None,
+    fixture_list: list[FixtureInfo] = FIXTURE_DATA,
 ):
     """Filter the fixtures based on supplied parameters.
 
@@ -127,12 +129,15 @@ def filter_fixtures(
             and (model := model_filter_list[0])
             and len(model.split("_")) == 3
         ):
-            # return exact match
+            # filter string includes hw and fw, return exact match
             return fixture_data.name == f"{model}.json"
         file_model_region = fixture_data.name.split("_")[0]
         file_model = file_model_region.split("(")[0]
         return file_model in model_filter
 
+    def _model_startswith_match(fixture_data: FixtureInfo, starts_with: str):
+        return fixture_data.name.startswith(starts_with)
+
     def _component_match(
         fixture_data: FixtureInfo, component_filter: str | ComponentFilter
     ):
@@ -175,13 +180,17 @@ def filter_fixtures(
     filtered = []
     if protocol_filter is None:
         protocol_filter = {"IOT", "SMART"}
-    for fixture_data in FIXTURE_DATA:
+    for fixture_data in fixture_list:
         if data_root_filter and data_root_filter not in fixture_data.data:
             continue
         if fixture_data.protocol not in protocol_filter:
             continue
         if model_filter is not None and not _model_match(fixture_data, model_filter):
             continue
+        if model_startswith_filter is not None and not _model_startswith_match(
+            fixture_data, model_startswith_filter
+        ):
+            continue
         if component_filter and not _component_match(fixture_data, component_filter):
             continue
         if device_type_filter and not _device_type_match(
@@ -191,8 +200,9 @@ def filter_fixtures(
 
         filtered.append(fixture_data)
 
-    print(f"# {desc}")
-    for value in filtered:
-        print(f"\t{value.name}")
+    if desc:
+        print(f"# {desc}")
+        for value in filtered:
+            print(f"\t{value.name}")
     filtered.sort()
     return filtered

tests/smart/modules/test_firmware.py

@@ -74,6 +74,7 @@ async def test_update_available_without_cloud(dev: SmartDevice):
         pytest.param(False, pytest.raises(KasaException), id="not-available"),
     ],
 )
+@pytest.mark.requires_dummy()
 async def test_firmware_update(
     dev: SmartDevice,
     mocker: MockerFixture,

tests/test_aestransport.py

@@ -29,6 +29,8 @@ from kasa.exceptions import (
 )
 from kasa.httpclient import HttpClient
 
+pytestmark = [pytest.mark.requires_dummy]
+
 DUMMY_QUERY = {"foobar": {"foo": "bar", "bar": "foo"}}
 
 key = b"8\x89\x02\xfa\xf5Xs\x1c\xa1 H\x9a\x82\xc7\xd9\t"

tests/test_bulb.py

@@ -32,7 +32,7 @@ from .conftest import (
 from .test_iotdevice import SYSINFO_SCHEMA
 
 
-@bulb
+@bulb_iot
 async def test_bulb_sysinfo(dev: Device):
     assert dev.sys_info is not None
     SYSINFO_SCHEMA_BULB(dev.sys_info)

tests/test_childdevice.py

@@ -125,8 +125,13 @@ async def test_parent_property(dev: Device):
 
 
 @has_children_smart
+@pytest.mark.requires_dummy()
 async def test_child_time(dev: Device, freezer: FrozenDateTimeFactory):
-    """Test a child device gets the time from it's parent module."""
+    """Test a child device gets the time from it's parent module.
+
+    This is excluded from real device testing as the test often fail if the
+    device time is not in the past.
+    """
     if not dev.children:
         pytest.skip(f"Device {dev} fixture does not have any children")
 

tests/test_cli.py

@@ -51,6 +51,10 @@ from .conftest import (
     turn_on,
 )
 
+# The cli tests should be testing the cli logic rather than a physical device
+# so mark the whole file for skipping with real devices.
+pytestmark = [pytest.mark.requires_dummy]
+
 
 @pytest.fixture()
 def runner():

tests/test_common_modules.py

@@ -1,7 +1,6 @@
 from datetime import datetime
 
 import pytest
-from freezegun.api import FrozenDateTimeFactory
 from pytest_mock import MockerFixture
 from zoneinfo import ZoneInfo
 
@@ -326,22 +325,38 @@ async def test_light_preset_save(dev: Device, mocker: MockerFixture):
     assert new_preset_state.color_temp == new_preset.color_temp
 
 
-async def test_set_time(dev: Device, freezer: FrozenDateTimeFactory):
+async def test_set_time(dev: Device):
     """Test setting the device time."""
-    freezer.move_to("2021-01-09 12:00:00+00:00")
     time_mod = dev.modules[Module.Time]
-    tz_info = time_mod.timezone
-    now = datetime.now(tz=tz_info)
-    now = now.replace(microsecond=0)
-    assert time_mod.time != now
 
-    await time_mod.set_time(now)
-    await dev.update()
-    assert time_mod.time == now
+    original_time = time_mod.time
+    original_timezone = time_mod.timezone
 
-    zone = ZoneInfo("Europe/Berlin")
-    now = datetime.now(tz=zone)
-    now = now.replace(microsecond=0)
-    await time_mod.set_time(now)
-    await dev.update()
-    assert time_mod.time == now
+    test_time = datetime.fromisoformat("2021-01-09 12:00:00+00:00")
+    test_time = test_time.astimezone(original_timezone)
+
+    try:
+        assert time_mod.time != test_time
+
+        await time_mod.set_time(test_time)
+        await dev.update()
+        assert time_mod.time == test_time
+
+        if (
+            isinstance(original_timezone, ZoneInfo)
+            and original_timezone.key != "Europe/Berlin"
+        ):
+            test_zonezone = ZoneInfo("Europe/Berlin")
+        else:
+            test_zonezone = ZoneInfo("Europe/London")
+
+        # Just update the timezone
+        new_time = time_mod.time.astimezone(test_zonezone)
+        await time_mod.set_time(new_time)
+        await dev.update()
+        assert time_mod.time == new_time
+    finally:
+        # Reset back to the original
+        await time_mod.set_time(original_time)
+        await dev.update()
+        assert time_mod.time == original_time

tests/test_device_factory.py

@@ -35,6 +35,10 @@ from kasa.discover import DiscoveryResult
 
 from .conftest import DISCOVERY_MOCK_IP
 
+# Device Factory tests are not relevant for real devices which run against
+# a single device that has already been created via the factory.
+pytestmark = [pytest.mark.requires_dummy]
+
 
 def _get_connection_type_device_class(discovery_info):
     if "result" in discovery_info:

tests/test_discovery.py

@@ -53,6 +53,9 @@ from .conftest import (
     wallswitch_iot,
 )
 
+# A physical device has to respond to discovery for the tests to work.
+pytestmark = [pytest.mark.requires_dummy]
+
 UNSUPPORTED = {
     "result": {
         "device_id": "xx",

tests/test_klapprotocol.py

@@ -32,6 +32,9 @@ from kasa.smartprotocol import SmartProtocol
 
 DUMMY_QUERY = {"foobar": {"foo": "bar", "bar": "foo"}}
 
+# Transport tests are not designed for real devices
+pytestmark = [pytest.mark.requires_dummy]
+
 
 class _mock_response:
     def __init__(self, status, content: bytes):

tests/test_protocol.py

@@ -687,10 +687,13 @@ def test_deprecated_protocol():
 @device_iot
 async def test_iot_queries_redaction(dev: IotDevice, caplog: pytest.LogCaptureFixture):
     """Test query sensitive info redaction."""
-    device_id = "123456789ABCDEF"
-    cast(FakeIotTransport, dev.protocol._transport).proto["system"]["get_sysinfo"][
-        "deviceId"
-    ] = device_id
+    if isinstance(dev.protocol._transport, FakeIotTransport):
+        device_id = "123456789ABCDEF"
+        cast(FakeIotTransport, dev.protocol._transport).proto["system"]["get_sysinfo"][
+            "deviceId"
+        ] = device_id
+    else:  # real device with --ip
+        device_id = dev.sys_info["deviceId"]
 
     # Info no message logging
     caplog.set_level(logging.INFO)

tests/test_smartdevice.py

@@ -26,6 +26,7 @@ from .conftest import (
 
 
 @device_smart
+@pytest.mark.requires_dummy()
 async def test_try_get_response(dev: SmartDevice, caplog):
     mock_response: dict = {
         "get_device_info": SmartErrorCode.PARAMS_ERROR,
@@ -37,6 +38,7 @@ async def test_try_get_response(dev: SmartDevice, caplog):
 
 
 @device_smart
+@pytest.mark.requires_dummy()
 async def test_update_no_device_info(dev: SmartDevice, mocker: MockerFixture):
     mock_response: dict = {
         "get_device_usage": {},

tests/test_smartprotocol.py

@@ -1,5 +1,4 @@
 import logging
-from typing import cast
 
 import pytest
 import pytest_mock
@@ -420,10 +419,11 @@ async def test_smart_queries_redaction(
     dev: SmartDevice, caplog: pytest.LogCaptureFixture
 ):
     """Test query sensitive info redaction."""
-    device_id = "123456789ABCDEF"
-    cast(FakeSmartTransport, dev.protocol._transport).info["get_device_info"][
-        "device_id"
-    ] = device_id
+    if isinstance(dev.protocol._transport, FakeSmartTransport):
+        device_id = "123456789ABCDEF"
+        dev.protocol._transport.info["get_device_info"]["device_id"] = device_id
+    else:  # real device
+        device_id = dev.device_id
 
     # Info no message logging
     caplog.set_level(logging.INFO)

tests/test_sslaestransport.py

@@ -27,6 +27,9 @@ from kasa.experimental.sslaestransport import (
 from kasa.httpclient import HttpClient
 from kasa.protocol import DEFAULT_CREDENTIALS, get_default_credentials
 
+# Transport tests are not designed for real devices
+pytestmark = [pytest.mark.requires_dummy]
+
 MOCK_ADMIN_USER = get_default_credentials(DEFAULT_CREDENTIALS["TAPOCAMERA"]).username
 MOCK_PWD = "correct_pwd"  # noqa: S105
 MOCK_USER = "mock@example.com"