[skip changelog] Optimize a bit integration tests

Author: silvanocerza

test/conftest.py

@@ -54,8 +54,11 @@ def data_dir(tmpdir_factory):
     if platform.system() == "Windows":
         with tempfile.TemporaryDirectory() as tmp:
             yield tmp
+            shutil.rmtree(tmp, ignore_errors=True)
     else:
-        yield str(tmpdir_factory.mktemp("ArduinoTest"))
+        data = tmpdir_factory.mktemp("ArduinoTest")
+        yield str(data)
+        shutil.rmtree(data, ignore_errors=True)
 
 
 @pytest.fixture(scope="session")
@@ -64,7 +67,9 @@ def downloads_dir(tmpdir_factory):
     To save time and bandwidth, all the tests will access
     the same download cache folder.
     """
-    return str(tmpdir_factory.mktemp("ArduinoTest"))
+    download_dir = tmpdir_factory.mktemp("ArduinoTest")
+    yield str(download_dir)
+    shutil.rmtree(download_dir, ignore_errors=True)
 
 
 @pytest.fixture(scope="function")
@@ -74,7 +79,9 @@ def working_dir(tmpdir_factory):
     will be created before running each test and deleted
     at the end, this way all the tests work in isolation.
     """
-    return str(tmpdir_factory.mktemp("ArduinoTestWork"))
+    work_dir = tmpdir_factory.mktemp("ArduinoTestWork")
+    yield str(work_dir)
+    shutil.rmtree(work_dir, ignore_errors=True)
 
 
 @pytest.fixture(scope="function")
@@ -95,9 +102,12 @@ def run_command(pytestconfig, data_dir, downloads_dir, working_dir):
     }
     (Path(data_dir) / "packages").mkdir()
 
-    def _run(cmd_string, custom_working_dir=None):
+    def _run(cmd_string, custom_working_dir=None, custom_env=None):
+
         if not custom_working_dir:
             custom_working_dir = working_dir
+        if not custom_env:
+            custom_env = env
         cli_full_line = '"{}" {}'.format(cli_path, cmd_string)
         run_context = Context()
         # It might happen that we need to change directories between drives on Windows,
@@ -109,7 +119,7 @@ def _run(cmd_string, custom_working_dir=None):
         # It escapes spaces in the path using "\ " but it doesn't always work,
         # wrapping the path in quotation marks is the safest approach
         with run_context.prefix(f'{cd_command} "{custom_working_dir}"'):
-            return run_context.run(cli_full_line, echo=False, hide=True, warn=True, env=env)
+            return run_context.run(cli_full_line, echo=False, hide=True, warn=True, env=custom_env)
 
     return _run
 
@@ -195,3 +205,64 @@ def copy_sketch(working_dir):
     test_sketch_path = Path(working_dir) / "sketch_simple"
     shutil.copytree(sketch_path, test_sketch_path)
     yield str(test_sketch_path)
+
+
+@pytest.fixture(scope="function")
+def core_update_index(run_command, data_dir, downloads_dir, working_dir, tmpdir_factory):
+    """
+    To save time and bandwidth we install and cache cores indexes and copy them to each individual test environment
+    """
+
+    def _update_index():
+        index_dir = tmpdir_factory.getbasetemp() / "core-indexes"
+        if not index_dir.exists():
+            index_dir.mkdir()
+            env = {
+                "ARDUINO_DATA_DIR": str(index_dir),
+                "ARDUINO_DOWNLOADS_DIR": downloads_dir,
+            }
+            run_command("core update-index", working_dir, env)
+        shutil.copytree(index_dir, data_dir, dirs_exist_ok=True)
+
+    return _update_index
+
+
+@pytest.fixture(scope="function")
+def lib_update_index(run_command, data_dir, downloads_dir, working_dir, tmpdir_factory):
+    """
+    To save time and bandwidth we install and cache libraries indexes and copy them to each individual test environment
+    """
+
+    def _update_index():
+        index_dir = tmpdir_factory.getbasetemp() / "lib-indexes"
+        if not index_dir.exists():
+            index_dir.mkdir()
+            env = {
+                "ARDUINO_DATA_DIR": str(index_dir),
+                "ARDUINO_DOWNLOADS_DIR": downloads_dir,
+            }
+            run_command("lib update-index", working_dir, env)
+        shutil.copyfile(index_dir / "library_index.json", Path(data_dir) / "library_index.json")
+
+    return _update_index
+
+
+@pytest.fixture(scope="function")
+def core_install(run_command, data_dir, downloads_dir, working_dir, tmpdir_factory):
+    """
+    To save time and bandwidth we install and cache cores and copy them to each individual test environment
+    """
+    data_dir = Path(data_dir) / "packages"
+
+    def _install(core):
+        core_dir = tmpdir_factory.getbasetemp() / core.replace(":", "")
+        if not core_dir.exists():
+            core_dir.mkdir()
+            env = {
+                "ARDUINO_DATA_DIR": str(core_dir),
+                "ARDUINO_DOWNLOADS_DIR": downloads_dir,
+            }
+            run_command(f"core install {core}", working_dir, env)
+        shutil.copytree(core_dir / "packages", data_dir, dirs_exist_ok=True)
+
+    return _install

test/test_board.py

@@ -389,9 +389,8 @@
 """  # noqa: E501
 
 
-def test_board_list(run_command):
-    result = run_command("core update-index")
-    assert result.ok
+def test_board_list(run_command, core_update_index):
+    core_update_index()
     result = run_command("board list --format json")
     assert result.ok
     # check is a valid json and contains a list of ports
@@ -403,19 +402,17 @@ def test_board_list(run_command):
 
 
 @pytest.mark.skipif(running_on_ci(), reason="VMs have no serial ports")
-def test_board_listall(run_command):
-    assert run_command("core update-index")
+def test_board_listall(run_command, core_update_index):
+    core_update_index()
     result = run_command("board listall")
     assert result.ok
     assert ["Board", "Name", "FQBN"] == result.stdout.splitlines()[0].strip().split()
 
 
-def test_board_details(run_command):
-    result = run_command("core update-index")
-    assert result.ok
+def test_board_details(run_command, core_update_index, core_install):
+    core_update_index()
     # Download samd core pinned to 1.8.6
-    result = run_command("core install arduino:samd@1.8.6")
-    assert result.ok
+    core_install("arduino:samd@1.8.6")
 
     # Test board listall with and without showing hidden elements
     result = run_command("board listall MIPS --format json")
@@ -447,12 +444,10 @@ def test_board_details(run_command):
 
 
 # old `arduino-cli board details` did not need -b <fqbn> flag to work
-def test_board_details_old(run_command):
-    result = run_command("core update-index")
-    assert result.ok
+def test_board_details_old(run_command, core_update_index, core_install):
+    core_update_index()
     # Download samd core pinned to 1.8.6
-    result = run_command("core install arduino:samd@1.8.6")
-    assert result.ok
+    core_install("arduino:samd@1.8.6")
     result = run_command("board details arduino:samd:nano_33_iot --format json")
     assert result.ok
     # Sort everything before compare
@@ -472,24 +467,20 @@ def test_board_details_old(run_command):
         assert programmer in result["programmers"]
 
 
-def test_board_details_no_flags(run_command):
-    result = run_command("core update-index")
-    assert result.ok
+def test_board_details_no_flags(run_command, core_update_index, core_install):
+    core_update_index()
     # Download samd core pinned to 1.8.6
-    result = run_command("core install arduino:samd@1.8.6")
-    assert result.ok
+    core_install("arduino:samd@1.8.6")
     result = run_command("board details")
     assert not result.ok
     assert "Error getting board details: parsing fqbn: invalid fqbn:" in result.stderr
     assert result.stdout == ""
 
 
-def test_board_details_list_programmers_without_flag(run_command):
-    result = run_command("core update-index")
-    assert result.ok
+def test_board_details_list_programmers_without_flag(run_command, core_update_index, core_install):
+    core_update_index()
     # Download samd core pinned to 1.8.6
-    result = run_command("core install arduino:samd@1.8.6")
-    assert result.ok
+    core_install("arduino:samd@1.8.6")
     result = run_command("board details -b arduino:samd:nano_33_iot")
     assert result.ok
     lines = [l.strip() for l in result.stdout.splitlines()]
@@ -502,12 +493,10 @@ def test_board_details_list_programmers_without_flag(run_command):
     assert "sam_ice                                                                Atmel SAM-ICE" in lines
 
 
-def test_board_details_list_programmers_flag(run_command):
-    result = run_command("core update-index")
-    assert result.ok
+def test_board_details_list_programmers_flag(run_command, core_update_index, core_install):
+    core_update_index()
     # Download samd core pinned to 1.8.6
-    result = run_command("core install arduino:samd@1.8.6")
-    assert result.ok
+    core_install("arduino:samd@1.8.6")
     result = run_command("board details -b arduino:samd:nano_33_iot --list-programmers")
     assert result.ok
 

test/test_compile.py

@@ -21,28 +21,24 @@
 from .common import running_on_ci
 
 
-def test_compile_without_fqbn(run_command):
+def test_compile_without_fqbn(run_command, core_update_index, core_install):
     # Init the environment explicitly
-    result = run_command("core update-index")
-    assert result.ok
+    core_update_index()
 
     # Install Arduino AVR Boards
-    result = run_command("core install arduino:avr@1.8.3")
-    assert result.ok
+    core_install("arduino:avr@1.8.3")
 
     # Build sketch without FQBN
     result = run_command("compile")
     assert result.failed
 
 
-def test_compile_with_simple_sketch(run_command, data_dir, working_dir):
+def test_compile_with_simple_sketch(run_command, data_dir, working_dir, core_update_index, core_install):
     # Init the environment explicitly
-    result = run_command("core update-index")
-    assert result.ok
+    core_update_index()
 
     # Download latest AVR
-    result = run_command("core install arduino:avr")
-    assert result.ok
+    core_install("arduino:avr")
 
     sketch_name = "CompileIntegrationTest"
     sketch_path = os.path.join(data_dir, sketch_name)
@@ -65,12 +61,9 @@ def test_compile_with_simple_sketch(run_command, data_dir, working_dir):
 
     # let's test from the logs if the hex file produced by successful compile is moved to our sketch folder
     log_json = open(log_file_path, "r")
-    json_log_lines = log_json.readlines()
-    expected_trace_sequence = [
-        "Compile {sketch} for {fqbn} started".format(sketch=sketch_path, fqbn=fqbn),
-        "Compile {sketch} for {fqbn} successful".format(sketch=sketch_name, fqbn=fqbn),
-    ]
-    assert is_message_sequence_in_json_log_traces(expected_trace_sequence, json_log_lines)
+    traces = parse_json_traces(log_json.readlines())
+    assert f"Compile {sketch_path} for {fqbn} started" in traces
+    assert f"Compile {sketch_name} for {fqbn} successful" in traces
 
     # Test the --output-dir flag with absolute path
     target = os.path.join(data_dir, "test_dir")
@@ -87,14 +80,12 @@ def test_compile_with_simple_sketch(run_command, data_dir, working_dir):
     running_on_ci() and platform.system() == "Windows",
     reason="Test disabled on Github Actions Win VM until tmpdir inconsistent behavior bug is fixed",
 )
-def test_output_flag_default_path(run_command, data_dir, working_dir):
+def test_output_flag_default_path(run_command, data_dir, working_dir, core_update_index, core_install):
     # Init the environment explicitly
-    result = run_command("core update-index")
-    assert result.ok
+    core_update_index()
 
     # Install Arduino AVR Boards
-    result = run_command("core install arduino:avr@1.8.3")
-    assert result.ok
+    core_install("arduino:avr@1.8.3")
 
     # Create a test sketch
     sketch_path = os.path.join(data_dir, "test_output_flag_default_path")
@@ -109,14 +100,12 @@ def test_output_flag_default_path(run_command, data_dir, working_dir):
     assert os.path.exists(target) and os.path.isdir(target)
 
 
-def test_compile_with_sketch_with_symlink_selfloop(run_command, data_dir):
+def test_compile_with_sketch_with_symlink_selfloop(run_command, data_dir, core_update_index, core_install):
     # Init the environment explicitly
-    result = run_command("core update-index")
-    assert result.ok
+    core_update_index()
 
     # Install Arduino AVR Boards
-    result = run_command("core install arduino:avr@1.8.3")
-    assert result.ok
+    core_install("arduino:avr@1.8.3")
 
     sketch_name = "CompileIntegrationTestSymlinkSelfLoop"
     sketch_path = os.path.join(data_dir, sketch_name)
@@ -162,15 +151,13 @@ def test_compile_with_sketch_with_symlink_selfloop(run_command, data_dir):
 
 
 @pytest.mark.skipif(running_on_ci(), reason="VMs have no serial ports")
-def test_compile_and_upload_combo(run_command, data_dir, detected_boards):
+def test_compile_and_upload_combo(run_command, data_dir, detected_boards, core_update_index, core_install):
     # Init the environment explicitly
-    result = run_command("core update-index")
-    assert result.ok
+    core_update_index()
 
     # Install required core(s)
-    result = run_command("core install arduino:avr@1.8.3")
-    result = run_command("core install arduino:samd@1.8.7")
-    assert result.ok
+    core_install("arduino:avr@1.8.3")
+    core_install("arduino:samd@1.8.6")
 
     # Create a test sketch
     sketch_name = "CompileAndUploadIntegrationTest"
@@ -196,41 +183,35 @@ def run_test(s):
 
             # check from the logs if the bin file were uploaded on the current board
             log_json = open(log_file_path, "r")
-            json_log_lines = log_json.readlines()
-            expected_trace_sequence = [
-                "Compile {sketch} for {fqbn} started".format(sketch=sketch_path, fqbn=board.fqbn),
-                "Compile {sketch} for {fqbn} successful".format(sketch=sketch_name, fqbn=board.fqbn),
-                "Upload {sketch} on {fqbn} started".format(sketch=sketch_path, fqbn=board.fqbn),
-                "Upload {sketch} on {fqbn} successful".format(sketch=sketch_name, fqbn=board.fqbn),
-            ]
-            assert is_message_sequence_in_json_log_traces(expected_trace_sequence, json_log_lines)
+            traces = parse_json_traces(log_json.readlines())
+            assert f"Compile {sketch_path} for {board.fqbn} started" in traces
+            assert f"Compile {sketch_name} for {board.fqbn} successful" in traces
+            assert f"Upload {sketch_path} on {board.fqbn} started" in traces
+            assert "Upload successful" in traces
 
         run_test(sketch_path)
         run_test(sketch_main_file)
 
 
-def is_message_sequence_in_json_log_traces(message_sequence, log_json_lines):
+def parse_json_traces(log_json_lines):
     trace_entries = []
     for entry in log_json_lines:
         entry = json.loads(entry)
         if entry.get("level") == "trace":
-            if entry.get("msg") in message_sequence:
-                trace_entries.append(entry.get("msg"))
-    return message_sequence == trace_entries
+            trace_entries.append(entry.get("msg"))
+    return trace_entries
 
 
-def test_compile_blacklisted_sketchname(run_command, data_dir):
+def test_compile_blacklisted_sketchname(run_command, data_dir, core_update_index, core_install):
     """
     Compile should ignore folders named `RCS`, `.git` and the likes, but
     it should be ok for a sketch to be named like RCS.ino
     """
     # Init the environment explicitly
-    result = run_command("core update-index")
-    assert result.ok
+    core_update_index()
 
     # Install Arduino AVR Boards
-    result = run_command("core install arduino:avr@1.8.3")
-    assert result.ok
+    core_install("arduino:avr@1.8.3")
 
     sketch_name = "RCS"
     sketch_path = os.path.join(data_dir, sketch_name)

test/test_lib.py

@@ -15,9 +15,9 @@
 import simplejson as json
 
 
-def test_list(run_command):
+def test_list(run_command, core_update_index):
     # Init the environment explicitly
-    assert run_command("core update-index")
+    core_update_index()
 
     # When output is empty, nothing is printed out, no matter the output format
     result = run_command("lib list")