all: Bump version to 1.26.1.

Signed-off-by: Damien George <damien@micropython.org>

ports/esp32/.gitignore

@@ -1,2 +1 @@
-dependencies.lock
 managed_components/

ports/esp32/CMakeLists.txt

@@ -61,5 +61,21 @@ set(SDKCONFIG_DEFAULTS ${CMAKE_BINARY_DIR}/sdkconfig.combined)
 # Include main IDF cmake file.
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)
 
+# Generate individual dependencies.lock files based on chip target
+idf_build_set_property(DEPENDENCIES_LOCK lockfiles/dependencies.lock.${IDF_TARGET})
+
 # Define the project.
 project(micropython)
+
+# Check for lockfile changes and either warn or error depending on build type
+message("Checking lockfile contents...")
+execute_process(COMMAND git diff --exit-code lockfiles/ WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
+    RESULT_VARIABLE RES)
+if (RES)
+    # Maintainer builds (CI or autobuild runs) should fail if this has happened
+    if($ENV{MICROPY_MAINTAINER_BUILD})
+        message(FATAL_ERROR "Failing build as lockfiles are dirty (see above). Check that ESP-IDF versions match.")
+    else()
+        message(WARNING "Component lockfile contents have changed (see above). This may be due to building with a different ESP-IDF version. Please mention this output if reporting an issue with MicroPython.")
+    endif()
+endif()

ports/esp32/README.md

@@ -30,8 +30,9 @@ framework, aka SDK).  The ESP-IDF includes the libraries and RTOS needed to
 manage the ESP32 microcontroller, as well as a way to manage the required
 build environment and toolchains needed to build the firmware.
 
-The ESP-IDF changes quickly and MicroPython only supports certain versions.
-Currently MicroPython supports v5.2, v5.2.2, v5.3, v5.4, v5.4.1 and v5.4.2.
+The ESP-IDF changes quickly and MicroPython only supports certain versions. The
+current recommended version of ESP-IDF for MicroPython is v5.4.2. MicroPython
+also supports v5.2, v5.2.2, v5.3, v5.4 and v5.4.1.
 
 To install the ESP-IDF the full instructions can be found at the
 [Espressif Getting Started guide](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/get-started/index.html#installation-step-by-step).

ports/esp32/lockfiles/README.md

@@ -0,0 +1,12 @@
+# ESP-IDF Component Lockfiles
+
+This directory contains the exact versions of ESP-IDF components that have been
+used to build MicroPython. It is updated by the [component version
+solver](https://docs.espressif.com/projects/idf-component-manager/en/latest/guides/version_solver.html).
+
+Unless you change the `main/idf_component.yml` file for MicroPython ESP32, files
+in this directory should only change contents if you build using a different
+ESP-IDF version to the last time the file was updated.
+
+*Please do not commit changes to these files and submit PRs unless the PR needs
+to change versions of components or ESP-IDF.*

ports/esp32/lockfiles/dependencies.lock.esp32

@@ -0,0 +1,35 @@
+dependencies:
+  espressif/lan867x:
+    component_hash: 0ff9dae3affeff53811e7c8283e67c6d36dc0c03e3bc5102c0fba629e08bf6c4
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.3'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    targets:
+    - esp32
+    - esp32p4
+    version: 1.0.3
+  espressif/mdns:
+    component_hash: 46ee81d32fbf850462d8af1e83303389602f6a6a9eddd2a55104cb4c063858ed
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    version: 1.1.0
+  idf:
+    source:
+      type: idf
+    version: 5.4.2
+direct_dependencies:
+- espressif/lan867x
+- espressif/mdns
+- idf
+manifest_hash: 3b18b5bbac91c9fe5098d3759a37c84ed0828546d8cbc92e26e4c1698e689c8a
+target: esp32
+version: 2.0.0

ports/esp32/lockfiles/dependencies.lock.esp32c2

@@ -0,0 +1,21 @@
+dependencies:
+  espressif/mdns:
+    component_hash: 46ee81d32fbf850462d8af1e83303389602f6a6a9eddd2a55104cb4c063858ed
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    version: 1.1.0
+  idf:
+    source:
+      type: idf
+    version: 5.4.2
+direct_dependencies:
+- espressif/mdns
+- idf
+manifest_hash: 3b18b5bbac91c9fe5098d3759a37c84ed0828546d8cbc92e26e4c1698e689c8a
+target: esp32c2
+version: 2.0.0

ports/esp32/lockfiles/dependencies.lock.esp32c3

@@ -0,0 +1,21 @@
+dependencies:
+  espressif/mdns:
+    component_hash: 46ee81d32fbf850462d8af1e83303389602f6a6a9eddd2a55104cb4c063858ed
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    version: 1.1.0
+  idf:
+    source:
+      type: idf
+    version: 5.4.2
+direct_dependencies:
+- espressif/mdns
+- idf
+manifest_hash: 3b18b5bbac91c9fe5098d3759a37c84ed0828546d8cbc92e26e4c1698e689c8a
+target: esp32c3
+version: 2.0.0

ports/esp32/lockfiles/dependencies.lock.esp32c6

@@ -0,0 +1,21 @@
+dependencies:
+  espressif/mdns:
+    component_hash: 46ee81d32fbf850462d8af1e83303389602f6a6a9eddd2a55104cb4c063858ed
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    version: 1.1.0
+  idf:
+    source:
+      type: idf
+    version: 5.4.2
+direct_dependencies:
+- espressif/mdns
+- idf
+manifest_hash: 3b18b5bbac91c9fe5098d3759a37c84ed0828546d8cbc92e26e4c1698e689c8a
+target: esp32c6
+version: 2.0.0

ports/esp32/lockfiles/dependencies.lock.esp32s2

@@ -0,0 +1,50 @@
+dependencies:
+  espressif/esp_tinyusb:
+    component_hash: 96d232ced7afe1976119b62f7fbf1944a2a78b36228ff6f7b9318394ac1153cc
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    - name: espressif/tinyusb
+      registry_url: https://components.espressif.com
+      require: public
+      version: '>=0.14.2'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    version: 1.7.6~1
+  espressif/mdns:
+    component_hash: 46ee81d32fbf850462d8af1e83303389602f6a6a9eddd2a55104cb4c063858ed
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    version: 1.1.0
+  espressif/tinyusb:
+    component_hash: aa65639878f27a44d349044afd9c3fc134a92bd560874fdac1d836019b5c07ca
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    source:
+      registry_url: https://components.espressif.com
+      type: service
+    targets:
+    - esp32s2
+    - esp32s3
+    - esp32p4
+    version: 0.18.0~4
+  idf:
+    source:
+      type: idf
+    version: 5.4.2
+direct_dependencies:
+- espressif/esp_tinyusb
+- espressif/mdns
+- idf
+manifest_hash: 3b18b5bbac91c9fe5098d3759a37c84ed0828546d8cbc92e26e4c1698e689c8a
+target: esp32s2
+version: 2.0.0

ports/esp32/lockfiles/dependencies.lock.esp32s3

@@ -0,0 +1,50 @@
+dependencies:
+  espressif/esp_tinyusb:
+    component_hash: 96d232ced7afe1976119b62f7fbf1944a2a78b36228ff6f7b9318394ac1153cc
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    - name: espressif/tinyusb
+      registry_url: https://components.espressif.com
+      require: public
+      version: '>=0.14.2'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    version: 1.7.6~1
+  espressif/mdns:
+    component_hash: 46ee81d32fbf850462d8af1e83303389602f6a6a9eddd2a55104cb4c063858ed
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    version: 1.1.0
+  espressif/tinyusb:
+    component_hash: aa65639878f27a44d349044afd9c3fc134a92bd560874fdac1d836019b5c07ca
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    source:
+      registry_url: https://components.espressif.com
+      type: service
+    targets:
+    - esp32s2
+    - esp32s3
+    - esp32p4
+    version: 0.18.0~4
+  idf:
+    source:
+      type: idf
+    version: 5.4.2
+direct_dependencies:
+- espressif/esp_tinyusb
+- espressif/mdns
+- idf
+manifest_hash: 3b18b5bbac91c9fe5098d3759a37c84ed0828546d8cbc92e26e4c1698e689c8a
+target: esp32s3
+version: 2.0.0

ports/esp32/main/idf_component.yml

@@ -4,7 +4,7 @@ dependencies:
   espressif/esp_tinyusb:
     rules:
       - if: "target in [esp32s2, esp32s3]"
-    version: "~1.0.0"
+    version: "~1.7.6"
   espressif/lan867x:
     version: "~1.0.0"
     rules:

py/mpconfig.h

@@ -31,7 +31,7 @@
 // are unavailable.
 #define MICROPY_VERSION_MAJOR 1
 #define MICROPY_VERSION_MINOR 26
-#define MICROPY_VERSION_MICRO 0
+#define MICROPY_VERSION_MICRO 1
 #define MICROPY_VERSION_PRERELEASE 0
 
 // Combined version as a 32-bit number for convenience to allow version

shared/tinyusb/mp_usbd.h

@@ -43,7 +43,18 @@
 // Initialise TinyUSB device.
 static inline void mp_usbd_init_tud(void) {
     tusb_init();
-    tud_cdc_configure_fifo_t cfg = { .rx_persistent = 0, .tx_persistent = 1 };
+    tud_cdc_configure_fifo_t cfg = { .rx_persistent = 0,
+                                     .tx_persistent = 1,
+
+                                     // This config flag is unreleased in TinyUSB >v0.18.0
+                                     // but included in Espressif's TinyUSB component since v0.18.0~3
+                                     //
+                                     // Versioning issue reported as
+                                     // https://github.com/espressif/esp-usb/issues/236
+                                     #if TUSB_VERSION_NUMBER > 1800 || defined(ESP_PLATFORM)
+                                     .tx_overwritabe_if_not_connected = 1,
+                                     #endif
+    };
     tud_cdc_configure_fifo(&cfg);
 }
 

shared/tinyusb/mp_usbd_cdc.c

@@ -98,32 +98,45 @@ mp_uint_t mp_usbd_cdc_tx_strn(const char *str, mp_uint_t len) {
     if (!tusb_inited()) {
         return 0;
     }
+    mp_uint_t last_write = mp_hal_ticks_ms();
     size_t i = 0;
     while (i < len) {
         uint32_t n = len - i;
-        if (n > CFG_TUD_CDC_EP_BUFSIZE) {
-            n = CFG_TUD_CDC_EP_BUFSIZE;
-        }
-        if (tud_cdc_connected()) {
-            // If CDC port is connected but the buffer is full, wait for up to USC_CDC_TIMEOUT ms.
-            mp_uint_t t0 = mp_hal_ticks_ms();
-            while (n > tud_cdc_write_available() && (mp_uint_t)(mp_hal_ticks_ms() - t0) < MICROPY_HW_USB_CDC_TX_TIMEOUT) {
-                mp_event_wait_ms(1);
 
-                // Explicitly run the USB stack as the scheduler may be locked (eg we
-                // are in an interrupt handler), while there is data pending.
-                mp_usbd_task();
-            }
+        if (tud_cdc_connected()) {
             // Limit write to available space in tx buffer when connected.
+            //
+            // (If not connected then we write everything to the fifo, expecting
+            // it to overwrite old data so it will have latest data buffered
+            // when host connects.)
             n = MIN(n, tud_cdc_write_available());
-            if (n == 0) {
-                break;
-            }
         }
-        // When not connected we always write to usb fifo, ensuring it has latest data.
+
         uint32_t n2 = tud_cdc_write(str + i, n);
         tud_cdc_write_flush();
         i += n2;
+
+        if (i < len) {
+            if (n2 > 0) {
+                // reset the timeout each time we successfully write to the FIFO
+                last_write = mp_hal_ticks_ms();
+            } else {
+                if ((mp_uint_t)(mp_hal_ticks_ms() - last_write) >= MICROPY_HW_USB_CDC_TX_TIMEOUT) {
+                    break; // Timeout
+                }
+
+                if (tud_cdc_connected()) {
+                    // If we know we're connected then we can wait for host to make
+                    // more space
+                    mp_event_wait_ms(1);
+                }
+            }
+
+            // Always explicitly run the USB stack as the scheduler may be
+            // locked (eg we are in an interrupt handler), while there is data
+            // or a state change pending.
+            mp_usbd_task();
+        }
     }
     return i;
 }

tools/autobuild/autobuild.sh

@@ -48,6 +48,9 @@ git -C ${MICROPY_AUTOBUILD_MICROPYTHON_REPO}/lib/pico-sdk submodule update --ini
 ########################################
 # Build all firmware
 
+# Fail on some things which are warnings otherwise
+export MICROPY_MAINTAINER_BUILD=1
+
 pushd ${MICROPY_AUTOBUILD_MICROPYTHON_REPO}
 
 # build cross compiler

tools/ci.sh

@@ -9,6 +9,9 @@ fi
 # Ensure known OPEN_MAX (NO_FILES) limit.
 ulimit -n 1024
 
+# Fail on some things which are warnings otherwise
+export MICROPY_MAINTAINER_BUILD=1
+
 ########################################################################################
 # general helper functions
 
@@ -168,14 +171,17 @@ function ci_cc3200_build {
 ########################################################################################
 # ports/esp32
 
-# GitHub tag of ESP-IDF to use for CI (note: must be a tag or a branch)
-IDF_VER=v5.4.2
+# GitHub tag of ESP-IDF to use for CI, extracted from the esp32 dependency lockfile
+# This should end up as a tag name like vX.Y.Z
+# (note: This hacky parsing can be replaced with 'yq' once Ubuntu >=24.04 is in use)
+IDF_VER=v$(grep -A10 "idf:" ports/esp32/lockfiles/dependencies.lock.esp32 | grep "version:" | head -n1 | sed -E 's/ +version: //')
 PYTHON=$(command -v python3 2> /dev/null)
 PYTHON_VER=$(${PYTHON:-python} --version | cut -d' ' -f2)
 
 export IDF_CCACHE_ENABLE=1
 
 function ci_esp32_idf_setup {
+    echo "Using ESP-IDF version $IDF_VER"
     git clone --depth 1 --branch $IDF_VER https://github.com/espressif/esp-idf.git
     # doing a treeless clone isn't quite as good as --shallow-submodules, but it
     # is smaller than full clones and works when the submodule commit isn't a head.

tools/mpremote/mpremote/transport_serial.py

@@ -36,11 +36,37 @@
 # as a command line tool and a library for interacting with devices.
 
 import ast, io, os, re, struct, sys, time
+import serial
+import serial.tools.list_ports
 from errno import EPERM
 from .console import VT_ENABLED
 from .transport import TransportError, TransportExecError, Transport
 
 VID_ESPRESSIF = 0x303A  # Espressif Incorporated
+PID_ESPRESSIF_SERIAL_JTAG = 0x1001  # Serial/JTAG peripheral of ESP32-S3,C3,C6
+
+
+def has_espressif_dtr_quirk(devicename):
+    """ESP8266 and ESP32 dev boards use the DTR and RTS lines to trigger reset &
+    reset into bootloader mode. This can causes spurious reset issues on Windows.
+
+    Apply the quirk to any USB/Serial chip on Windows that isn't using the
+    Microsoft CDC-ACM driver, or to the integrated Espressif Serial/JTAG device.
+
+    Don't apply it to Espressif boards running TinyUSB, as TinyUSB uses DTR
+    to determine if the CDC port is open (and there's no spurious reset issue).
+    """
+    portinfo = list(serial.tools.list_ports.grep(devicename))  # type: ignore
+    if not portinfo:
+        return False
+
+    def port_attr(name):
+        return getattr(portinfo[0], name, None)
+
+    return (port_attr("vid"), port_attr("pid")) == (
+        VID_ESPRESSIF,
+        PID_ESPRESSIF_SERIAL_JTAG,
+    ) or port_attr("manufacturer") != "Microsoft"
 
 
 class SerialTransport(Transport):
@@ -52,9 +78,6 @@ class SerialTransport(Transport):
         self.device_name = device
         self.mounted = False
 
-        import serial
-        import serial.tools.list_ports
-
         # Set options, and exclusive if pyserial supports it
         serial_kwargs = {
             "baudrate": baudrate,
@@ -72,12 +95,7 @@ class SerialTransport(Transport):
                 elif os.name == "nt":
                     self.serial = serial.Serial(**serial_kwargs)
                     self.serial.port = device
-                    portinfo = list(serial.tools.list_ports.grep(device))  # type: ignore
-                    if portinfo and (
-                        getattr(portinfo[0], "vid", 0) == VID_ESPRESSIF
-                        or getattr(portinfo[0], "manufacturer", "") != "Microsoft"
-                    ):
-                        # ESP8266/ESP32 boards use RTS/CTS for flashing and boot mode selection.
+                    if has_espressif_dtr_quirk(device):
                         # DTR False: to avoid using the reset button will hang the MCU in bootloader mode
                         # RTS False: to prevent pulses on rts on serial.close() that would POWERON_RESET an ESPxx
                         self.serial.dtr = False  # DTR False = gpio0 High = Normal boot