all: Bump version to 1.11.

Otherwise flushing and disabling the D-cache will give a hard-fault when
SDRAM is used.

Fixes #4818.

.travis.yml

@@ -1,4 +1,5 @@
 # global options
+dist: xenial
 language:
   - c
 compiler:
@@ -23,9 +24,10 @@ jobs:
       env: NAME="stm32 port build"
       install:
         # need newer gcc version for Cortex-M7 support
-        - sudo add-apt-repository -y ppa:terry.guo/gcc-arm-embedded
+        - sudo add-apt-repository -y ppa:team-gcc-arm-embedded/ppa
         - sudo apt-get update -qq || true
-        - sudo apt-get install --allow-unauthenticated gcc-arm-none-eabi
+        - sudo apt-get install gcc-arm-embedded
+        - sudo apt-get install libnewlib-arm-none-eabi
         - arm-none-eabi-gcc --version
       script:
         - make ${MAKEOPTS} -C mpy-cross
@@ -38,12 +40,11 @@ jobs:
     - stage: test
       env: NAME="qemu-arm port build and tests"
       install:
-        # need newer gcc version for nano.specs
-        - sudo add-apt-repository -y ppa:terry.guo/gcc-arm-embedded
-        - sudo apt-get update -qq || true
-        - sudo apt-get install --allow-unauthenticated gcc-arm-none-eabi
+        - sudo apt-get install gcc-arm-none-eabi
+        - sudo apt-get install libnewlib-arm-none-eabi
         - sudo apt-get install qemu-system
         - arm-none-eabi-gcc --version
+        - qemu-system-arm --version
       script:
         - make ${MAKEOPTS} -C mpy-cross
         - make ${MAKEOPTS} -C ports/qemu-arm -f Makefile.test test
@@ -54,8 +55,6 @@ jobs:
     - stage: test
       env: NAME="unix coverage build and tests"
       install:
-        # a specific urllib3 version is needed for requests and cpp-coveralls to work together
-        - sudo pip install -Iv urllib3==1.22
         - sudo pip install cpp-coveralls
         - gcc --version
         - python3 --version
@@ -67,7 +66,8 @@ jobs:
         - (cd tests && MICROPY_CPYTHON3=python3 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests)
         - (cd tests && MICROPY_CPYTHON3=python3 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests -d thread)
         - (cd tests && MICROPY_CPYTHON3=python3 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests --emit native)
-        - (cd tests && MICROPY_CPYTHON3=python3 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests --via-mpy -d basics float)
+        - (cd tests && MICROPY_CPYTHON3=python3 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests --via-mpy -d basics float micropython)
+        - (cd tests && MICROPY_CPYTHON3=python3 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests --via-mpy --emit native -d basics float micropython)
         # test when input script comes from stdin
         - cat tests/basics/0prelim.py | ports/unix/micropython_coverage | grep -q 'abc'
         # run coveralls coverage analysis (try to, even if some builds/tests failed)
@@ -117,10 +117,8 @@ jobs:
     - stage: test
       env: NAME="nrf port build"
       install:
-        # need newer gcc version to support variables in linker script
-        - sudo add-apt-repository -y ppa:team-gcc-arm-embedded/ppa
-        - sudo apt-get update -qq || true
-        - sudo apt-get install gcc-arm-embedded
+        - sudo apt-get install gcc-arm-none-eabi
+        - sudo apt-get install libnewlib-arm-none-eabi
         - arm-none-eabi-gcc --version
       script:
         - make ${MAKEOPTS} -C ports/nrf
@@ -130,6 +128,7 @@ jobs:
       env: NAME="bare-arm and minimal ports build"
       install:
         - sudo apt-get install gcc-arm-none-eabi
+        - sudo apt-get install libnewlib-arm-none-eabi
         - arm-none-eabi-gcc --version
       script:
         - make ${MAKEOPTS} -C ports/bare-arm
@@ -145,6 +144,7 @@ jobs:
       env: NAME="cc3200 port build"
       install:
         - sudo apt-get install gcc-arm-none-eabi
+        - sudo apt-get install libnewlib-arm-none-eabi
       script:
         - make ${MAKEOPTS} -C ports/cc3200 BTARGET=application BTYPE=release
         - make ${MAKEOPTS} -C ports/cc3200 BTARGET=bootloader  BTYPE=release
@@ -154,5 +154,6 @@ jobs:
       env: NAME="teensy port build"
       install:
         - sudo apt-get install gcc-arm-none-eabi
+        - sudo apt-get install libnewlib-arm-none-eabi
       script:
         - make ${MAKEOPTS} -C ports/teensy

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2013, 2014 Damien P. George
+Copyright (c) 2013-2019 Damien P. George
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -60,7 +60,9 @@ Additional components:
 The subdirectories above may include READMEs with additional info.
 
 "make" is used to build the components, or "gmake" on BSD-based systems.
-You will also need bash, gcc, and Python (at least 2.7 or 3.3).
+You will also need bash, gcc, and Python 3.3+ available as the command `python3`
+(if your system only has Python 2.7 then invoke make with the additional option
+`PYTHON=python2`).
 
 The Unix version
 ----------------

docs/conf.py

@@ -74,7 +74,7 @@ copyright = '2014-2019, Damien P. George, Paul Sokolovsky, and contributors'
 #
 # We don't follow "The short X.Y version" vs "The full version, including alpha/beta/rc tags"
 # breakdown, so use the same version identifier for both to avoid confusion.
-version = release = '1.10'
+version = release = '1.11'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/develop/cmodules.rst

@@ -0,0 +1,174 @@
+MicroPython external C modules
+==============================
+
+When developing modules for use with MicroPython you may find you run into
+limitations with the Python environment, often due to an inability to access
+certain hardware resources or Python speed limitations.
+
+If your limitations can't be resolved with suggestions in :ref:`speed_python`,
+writing some or all of your module in C is a viable option.
+
+If your module is designed to access or work with commonly available
+hardware or libraries please consider implementing it inside the MicroPython
+source tree alongside similar modules and submitting it as a pull request.
+If however you're targeting obscure or proprietary systems it may make
+more sense to keep this external to the main MicroPython repository.
+
+This chapter describes how to compile such external modules into the
+MicroPython executable or firmware image.
+
+
+Structure of an external C module
+---------------------------------
+
+A MicroPython user C module is a directory with the following files:
+
+* ``*.c`` and/or ``*.h`` source code files for your module.
+
+  These will typically include the low level functionality being implemented and
+  the MicroPython binding functions to expose the functions and module(s).
+
+  Currently the best reference for writing these functions/modules is
+  to find similar modules within the MicroPython tree and use them as examples.
+
+* ``micropython.mk`` contains the Makefile fragment for this module.
+
+  ``$(USERMOD_DIR)`` is available in ``micropython.mk`` as the path to your
+  module directory. As it's redefined for each c module, is should be expanded
+  in your ``micropython.mk`` to a local make variable,
+  eg ``EXAMPLE_MOD_DIR := $(USERMOD_DIR)``
+
+  Your ``micropython.mk`` must add your modules C files relative to your
+  expanded copy of ``$(USERMOD_DIR)`` to ``SRC_USERMOD``, eg
+  ``SRC_USERMOD += $(EXAMPLE_MOD_DIR)/example.c``
+
+  If you have custom ``CFLAGS`` settings or include folders to define, these
+  should be added to ``CFLAGS_USERMOD``.
+
+  See below for full usage example.
+
+
+Basic Example
+-------------
+
+This simple module named ``example`` provides a single function
+``example.add_ints(a, b)`` which adds the two integer args together and returns
+the result.
+
+Directory::
+
+    example/
+    ├── example.c
+    └── micropython.mk
+
+
+``example.c``
+
+.. code-block:: c
+
+    // Include required definitions first.
+    #include "py/obj.h"
+    #include "py/runtime.h"
+    #include "py/builtin.h"
+
+    // This is the function which will be called from Python as example.add_ints(a, b).
+    STATIC mp_obj_t example_add_ints(mp_obj_t a_obj, mp_obj_t b_obj) {
+        // Extract the ints from the micropython input objects
+        int a = mp_obj_get_int(a_obj);
+        int b = mp_obj_get_int(b_obj);
+
+        // Calculate the addition and convert to MicroPython object.
+        return mp_obj_new_int(a + b);
+    }
+    // Define a Python reference to the function above
+    STATIC MP_DEFINE_CONST_FUN_OBJ_2(example_add_ints_obj, example_add_ints);
+
+    // Define all properties of the example module.
+    // Table entries are key/value pairs of the attribute name (a string)
+    // and the MicroPython object reference.
+    // All identifiers and strings are written as MP_QSTR_xxx and will be
+    // optimized to word-sized integers by the build system (interned strings).
+    STATIC const mp_rom_map_elem_t example_module_globals_table[] = {
+        { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_example) },
+        { MP_ROM_QSTR(MP_QSTR_add_ints), MP_ROM_PTR(&example_add_ints_obj) },
+    };
+    STATIC MP_DEFINE_CONST_DICT(example_module_globals, example_module_globals_table);
+
+    // Define module object.
+    const mp_obj_module_t example_user_cmodule = {
+        .base = { &mp_type_module },
+        .globals = (mp_obj_dict_t*)&example_module_globals,
+    };
+
+    // Register the module to make it available in Python
+    MP_REGISTER_MODULE(MP_QSTR_example, example_user_cmodule, MODULE_EXAMPLE_ENABLED);
+
+
+``micropython.mk``
+
+.. code-block:: make
+
+    EXAMPLE_MOD_DIR := $(USERMOD_DIR)
+
+    # Add all C files to SRC_USERMOD.
+    SRC_USERMOD += $(EXAMPLE_MOD_DIR)/example.c
+
+    # We can add our module folder to include paths if needed
+    # This is not actually needed in this example.
+    CFLAGS_USERMOD += -I$(EXAMPLE_MOD_DIR)
+
+Finally you will need to define ``MODULE_EXAMPLE_ENABLED`` to 1. This
+can be done by adding ``CFLAGS_EXTRA=-DMODULE_EXAMPLE_ENABLED=1`` to
+the ``make`` command, or editing ``mpconfigport.h`` or
+``mpconfigboard.h`` to add
+
+.. code-block:: c
+
+    #define MODULE_EXAMPLE_ENABLED (1)
+
+Note that the exact method depends on the port as they have different
+structures. If not done correctly it will compile but importing will
+fail to find the module.
+
+
+Compiling the cmodule into MicroPython
+--------------------------------------
+
+To build such a module, compile MicroPython (see `getting started
+<https://github.com/micropython/micropython/wiki/Getting-Started>`_) with an
+extra ``make`` flag named ``USER_C_MODULES`` set to the directory containing
+all modules you want included (not to the module itself). For example:
+
+
+Directory::
+
+    my_project/
+    ├── modules/
+    │   └──example/
+    │       ├──example.c
+    │       └──micropython.mk
+    └── micropython/
+        ├──ports/
+       ... ├──stm32/
+          ...
+
+Building for stm32 port:
+
+.. code-block:: bash
+
+    cd my_project/micropython/ports/stm32
+    make USER_C_MODULES=../../../modules CFLAGS_EXTRA=-DMODULE_EXAMPLE_ENABLED=1 all
+
+
+Module usage in MicroPython
+---------------------------
+
+Once built into your copy of MicroPython, the module implemented
+in ``example.c`` above can now be accessed in Python just
+like any other builtin module, eg
+
+.. code-block:: python
+
+    import example
+    print(example.add_ints(1, 3))
+    # should display 4

docs/develop/index.rst

@@ -0,0 +1,12 @@
+Developing and building MicroPython
+===================================
+
+This chapter describes some options for extending MicroPython in C. Note
+that it doesn't aim to be a complete guide for developing with MicroPython.
+See the `getting started guide
+<https://github.com/micropython/micropython/wiki/Getting-Started>`_ for further information.
+
+.. toctree::
+   :maxdepth: 1
+
+   cmodules.rst

docs/esp32/quickref.rst

@@ -128,6 +128,8 @@ with timer ID of -1::
 
 The period is in milliseconds.
 
+.. _Pins_and_GPIO:
+
 Pins and GPIO
 -------------
 
@@ -161,6 +163,9 @@ Notes:
 
 * Pins 34-39 are input only, and also do not have internal pull-up resistors
 
+* The pull value of some pins can be set to ``Pin.PULL_HOLD`` to reduce power
+  consumption during deepsleep.
+
 PWM (pulse width modulation)
 ----------------------------
 
@@ -271,8 +276,13 @@ class::
 Hardware SPI bus
 ----------------
 
-There are two hardware SPI channels that allow faster (up to 80Mhz)
-transmission rates, but are only supported on a subset of pins.
+There are two hardware SPI channels that allow faster transmission
+rates (up to 80Mhz). These may be used on any IO pins that support the
+required direction and are otherwise unused (see :ref:`Pins_and_GPIO`)
+but if they are not configured to their default pins then they need to
+pass through an extra layer of GPIO multiplexing, which can impact
+their reliability at high speeds. Hardware SPI channels are limited
+to 40MHz when used on pins other than the default ones listed below.
 
 =====  ===========  ============
 \      HSPI (id=1)   VSPI (id=2)
@@ -337,6 +347,15 @@ Notes:
 * Calling ``deepsleep()`` without an argument will put the device to sleep
   indefinitely
 * A software reset does not change the reset cause
+* There may be some leakage current flowing through enabled internal pullups.
+  To further reduce power consumption it is possible to disable the internal pullups::
+
+    p1 = Pin(4, Pin.IN, Pin.PULL_HOLD)
+    
+  After leaving deepsleep it may be necessary to un-hold the pin explicitly (e.g. if
+  it is an output pin) via::
+    
+    p1 = Pin(4, Pin.OUT, None)
 
 OneWire driver
 --------------
@@ -421,7 +440,7 @@ Note that TouchPads can be used to wake an ESP32 from sleep::
     t = TouchPad(Pin(14))
     t.config(500)               # configure the threshold at which the pin is considered touched
     esp32.wake_on_touch(True)
-    machine.sleep()             # put the MCU to sleep until a touchpad is touched
+    machine.lightsleep()        # put the MCU to sleep until a touchpad is touched
 
 For more details on touchpads refer to `Espressif Touch Sensor
 <https://docs.espressif.com/projects/esp-idf/en/latest/api-reference/peripherals/touch_pad.html>`_.

docs/esp8266/tutorial/apa102.rst

@@ -0,0 +1,91 @@
+Controlling APA102 LEDs
+=======================
+
+APA102 LEDs, also known as DotStar LEDs, are individually addressable
+full-colour RGB LEDs, generally in a string formation. They differ from
+NeoPixels in that they require two pins to control - both a Clock and Data pin.
+They can operate at a much higher data and PWM frequencies than NeoPixels and
+are more suitable for persistence-of-vision effects.
+
+To create an APA102 object do the following::
+
+    >>> import machine, apa102
+    >>> strip = apa102.APA102(machine.Pin(5), machine.Pin(4), 60)
+
+This configures an 60 pixel APA102 strip with clock on GPIO5 and data on GPIO4.
+You can adjust the pin numbers and the number of pixels to suit your needs.
+
+The RGB colour data, as well as a brightness level, is sent to the APA102 in a
+certain order.  Usually this is ``(Red, Green, Blue, Brightness)``.
+If you are using one of the newer APA102C LEDs the green and blue are swapped,
+so the order is ``(Red, Blue, Green, Brightness)``.
+The APA102 has more of a square lens while the APA102C has more of a round one.
+If you are using a APA102C strip and would prefer to provide colours in RGB
+order instead of RBG, you can customise the tuple colour order like so::
+
+    >>> strip.ORDER = (0, 2, 1, 3)
+
+To set the colour of pixels use::
+
+    >>> strip[0] = (255, 255, 255, 31) # set to white, full brightness
+    >>> strip[1] = (255, 0, 0, 31) # set to red, full brightness
+    >>> strip[2] = (0, 255, 0, 15) # set to green, half brightness
+    >>> strip[3] = (0, 0, 255, 7)  # set to blue, quarter brightness
+
+Use the ``write()`` method to output the colours to the LEDs::
+
+    >>> strip.write()
+
+Demonstration::
+
+    import time
+    import machine, apa102
+
+    # 1M strip with 60 LEDs
+    strip = apa102.APA102(machine.Pin(5), machine.Pin(4), 60)
+
+    brightness = 1  # 0 is off, 1 is dim, 31 is max
+
+    # Helper for converting 0-255 offset to a colour tuple
+    def wheel(offset, brightness):
+        # The colours are a transition r - g - b - back to r
+        offset = 255 - offset
+        if offset < 85:
+            return (255 - offset * 3, 0, offset * 3, brightness)
+        if offset < 170:
+            offset -= 85
+            return (0, offset * 3, 255 - offset * 3, brightness)
+        offset -= 170
+        return (offset * 3, 255 - offset * 3, 0, brightness)
+
+    # Demo 1: RGB RGB RGB
+    red = 0xff0000
+    green = red >> 8
+    blue = red >> 16
+    for i in range(strip.n):
+        colour = red >> (i % 3) * 8
+        strip[i] = ((colour & red) >> 16, (colour & green) >> 8, (colour & blue), brightness)
+    strip.write()
+
+    # Demo 2: Show all colours of the rainbow
+    for i in range(strip.n):
+        strip[i] = wheel((i * 256 // strip.n) % 255, brightness)
+    strip.write()
+
+    # Demo 3: Fade all pixels together through rainbow colours, offset each pixel
+    for r in range(5):
+        for n in range(256):
+            for i in range(strip.n):
+                strip[i] = wheel(((i * 256 // strip.n) + n) & 255, brightness)
+            strip.write()
+        time.sleep_ms(25)
+
+    # Demo 4: Same colour, different brightness levels
+    for b in range(31,-1,-1):
+        strip[0] = (255, 153, 0, b)
+        strip.write()
+        time.sleep_ms(250)
+
+    # End: Turn off all the LEDs
+    strip.fill((0, 0, 0, 0))
+    strip.write()

docs/esp8266/tutorial/index.rst

@@ -29,5 +29,6 @@ to `<https://www.python.org>`__.
    powerctrl.rst
    onewire.rst
    neopixel.rst
+   apa102.rst
    dht.rst
    nextsteps.rst

docs/index.rst

@@ -6,6 +6,7 @@ MicroPython documentation and references
     library/index.rst
     reference/index.rst
     genrst/index.rst
+    develop/index.rst
     license.rst
     pyboard/quickref.rst
     esp8266/quickref.rst

docs/library/machine.I2C.rst

@@ -131,6 +131,20 @@ operations that target a given slave device.
    generated at the end of the transfer, even if a NACK is received.
    The function returns the number of ACKs that were received.
 
+.. method:: I2C.writevto(addr, vector, stop=True)
+
+   Write the bytes contained in *vector* to the slave specified by *addr*.
+   *vector* should be a tuple or list of objects with the buffer protocol.
+   The *addr* is sent once and then the bytes from each object in *vector*
+   are written out sequentially.  The objects in *vector* may be zero bytes
+   in length in which case they don't contribute to the output.
+
+   If a NACK is received following the write of a byte from one of the
+   objects in *vector* then the remaining bytes, and any remaining objects,
+   are not sent.  If *stop* is true then a STOP condition is generated at
+   the end of the transfer, even if a NACK is received.  The function
+   returns the number of ACKs that were received.
+
 Memory operations
 -----------------
 

docs/library/machine.Pin.rst

@@ -235,6 +235,7 @@ not all constants are available on all ports.
 
 .. data:: Pin.PULL_UP
           Pin.PULL_DOWN
+          Pin.PULL_HOLD
 
    Selects whether there is a pull up/down resistor.  Use the value
    ``None`` for no pull.

docs/library/machine.rst

@@ -63,16 +63,31 @@ Power related functions
 
 .. function:: sleep()
 
-   Stops the CPU and disables all peripherals except for WLAN. Execution is resumed from
-   the point where the sleep was requested. For wake up to actually happen, wake sources
-   should be configured first.
+   .. note:: This function is deprecated, use `lightsleep()` instead with no arguments.
 
-.. function:: deepsleep()
+.. function:: lightsleep([time_ms])
+              deepsleep([time_ms])
 
-   Stops the CPU and all peripherals (including networking interfaces, if any). Execution
-   is resumed from the main script, just as with a reset. The reset cause can be checked
-   to know that we are coming from `machine.DEEPSLEEP`. For wake up to actually happen,
-   wake sources should be configured first, like `Pin` change or `RTC` timeout.
+   Stops execution in an attempt to enter a low power state.
+
+   If *time_ms* is specified then this will be the maximum time in milliseconds that
+   the sleep will last for.  Otherwise the sleep can last indefinitely.
+
+   With or without a timout, execution may resume at any time if there are events
+   that require processing.  Such events, or wake sources, should be configured before
+   sleeping, like `Pin` change or `RTC` timeout.
+
+   The precise behaviour and power-saving capabilities of lightsleep and deepsleep is
+   highly dependent on the underlying hardware, but the general properties are:
+
+   * A lightsleep has full RAM and state retention.  Upon wake execution is resumed
+     from the point where the sleep was requested, with all subsystems operational.
+
+   * A deepsleep may not retain RAM or any other state of the system (for example
+     peripherals or network interfaces).  Upon wake execution is resumed from the main
+     script, similar to a hard or power-on reset. The `reset_cause()` function will
+     return `machine.DEEPSLEEP` and this can be used to distinguish a deepsleep wake
+     from other resets.
 
 .. function:: wake_reason()
 

docs/library/pyb.UART.rst

@@ -69,7 +69,7 @@ Constructors
 Methods
 -------
 
-.. method:: UART.init(baudrate, bits=8, parity=None, stop=1, \*, timeout=1000, flow=0, timeout_char=0, read_buf_len=64)
+.. method:: UART.init(baudrate, bits=8, parity=None, stop=1, \*, timeout=0, flow=0, timeout_char=0, read_buf_len=64)
 
    Initialise the UART bus with the given parameters:
 

docs/library/ucryptolib.rst

@@ -22,9 +22,11 @@ Classes
             * *mode* is:
 
                 * ``1`` (or ``ucryptolib.MODE_ECB`` if it exists) for Electronic Code Book (ECB).
-                * ``2`` (or ``ucryptolib.MODE_CBC`` if it exists) for Cipher Block Chaining (CBC)
+                * ``2`` (or ``ucryptolib.MODE_CBC`` if it exists) for Cipher Block Chaining (CBC).
+                * ``6`` (or ``ucryptolib.MODE_CTR`` if it exists) for Counter mode (CTR).
 
             * *IV* is an initialization vector for CBC mode.
+            * For Counter mode, *IV* is the initial value for the counter.
 
     .. method:: encrypt(in_buf, [out_buf])
 

docs/library/uos.rst

@@ -115,7 +115,8 @@ Terminal redirection and duplication
 .. function:: dupterm(stream_object, index=0)
 
    Duplicate or switch the MicroPython terminal (the REPL) on the given `stream`-like
-   object. The *stream_object* argument must implement the ``readinto()`` and
+   object. The *stream_object* argument must be a native stream object, or derive
+   from ``uio.IOBase`` and implement the ``readinto()`` and
    ``write()`` methods.  The stream should be in non-blocking mode and
    ``readinto()`` should return ``None`` if there is no data available for reading.
 

docs/library/ure.rst

@@ -175,7 +175,7 @@ Match objects
 Match objects as returned by `match()` and `search()` methods, and passed
 to the replacement function in `sub()`.
 
-.. method:: match.group([index])
+.. method:: match.group(index)
 
    Return matching (sub)string. *index* is 0 for entire match,
    1 and above for each capturing group. Only numeric groups are supported.

docs/pyboard/hardware/index.rst

@@ -3,9 +3,12 @@ The pyboard hardware
 
 For the pyboard:
 
-* `PYBv1.0 schematics and layout <http://micropython.org/resources/PYBv10b.pdf>`_ (2.4MiB PDF)
-* `PYBv1.0 metric dimensions <http://micropython.org/resources/PYBv10b-metric-dimensions.pdf>`_ (360KiB PDF)
-* `PYBv1.0 imperial dimensions <http://micropython.org/resources/PYBv10b-imperial-dimensions.pdf>`_ (360KiB PDF)
+* v1.1
+    * `PYBv1.1 schematics and layout <https://micropython.org/resources/PYBv11.pdf>`_ (2.9MiB PDF)
+* v1.0
+    * `PYBv1.0 schematics and layout <http://micropython.org/resources/PYBv10b.pdf>`_ (2.4MiB PDF)
+    * `PYBv1.0 metric dimensions <http://micropython.org/resources/PYBv10b-metric-dimensions.pdf>`_ (360KiB PDF)
+    * `PYBv1.0 imperial dimensions <http://micropython.org/resources/PYBv10b-imperial-dimensions.pdf>`_ (360KiB PDF)
 
 For the official skin modules:
 

docs/pyboard/quickref.rst

@@ -3,22 +3,22 @@
 Quick reference for the pyboard
 ===============================
 
-The below pinout is for PYBv1.0.  You can also view pinouts for
+The below pinout is for PYBv1.1.  You can also view pinouts for
 other versions of the pyboard:
-`PYBv1.1 <http://micropython.org/resources/pybv11-pinout.jpg>`__
+`PYBv1.0 <http://micropython.org/resources/pybv10-pinout.jpg>`__
 or `PYBLITEv1.0-AC <http://micropython.org/resources/pyblitev10ac-pinout.jpg>`__
 or `PYBLITEv1.0 <http://micropython.org/resources/pyblitev10-pinout.jpg>`__.
 
 .. only:: not latex
 
-   .. image:: http://micropython.org/resources/pybv10-pinout.jpg
-      :alt: PYBv1.0 pinout
+   .. image:: http://micropython.org/resources/pybv11-pinout.jpg
+      :alt: PYBv1.1 pinout
       :width: 700px
 
 .. only:: latex
 
-   .. image:: http://micropython.org/resources/pybv10-pinout-800px.jpg
-      :alt: PYBv1.0 pinout
+   .. image:: http://micropython.org/resources/pybv11-pinout-800px.jpg
+      :alt: PYBv1.1 pinout
 
 Below is a quick reference for the pyboard.  If it is your first time working with
 this board please consider reading the following sections first:
@@ -199,16 +199,25 @@ See :ref:`pyb.SPI <pyb.SPI>`. ::
 I2C bus
 -------
 
-See :ref:`pyb.I2C <pyb.I2C>`. ::
+Hardware I2C is available on the X and Y halves of the pyboard via ``I2C('X')``
+and ``I2C('Y')``.  Alternatively pass in the integer identifier of the peripheral,
+eg ``I2C(1)``.  Software I2C is also available by explicitly specifying the
+``scl`` and ``sda`` pins instead of the bus name.  For more details see
+:ref:`machine.I2C <machine.I2C>`. ::
 
-    from pyb import I2C
+    from machine import I2C
 
-    i2c = I2C(1, I2C.MASTER, baudrate=100000)
-    i2c.scan() # returns list of slave addresses
-    i2c.send('hello', 0x42) # send 5 bytes to slave with address 0x42
-    i2c.recv(5, 0x42) # receive 5 bytes from slave
-    i2c.mem_read(2, 0x42, 0x10) # read 2 bytes from slave 0x42, slave memory 0x10
-    i2c.mem_write('xy', 0x42, 0x10) # write 2 bytes to slave 0x42, slave memory 0x10
+    i2c = I2C('X', freq=400000)                 # create hardware I2c object
+    i2c = I2C(scl='X1', sda='X2', freq=100000)  # create software I2C object
+
+    i2c.scan()                          # returns list of slave addresses
+    i2c.writeto(0x42, 'hello')          # write 5 bytes to slave with address 0x42
+    i2c.readfrom(0x42, 5)               # read 5 bytes from slave
+
+    i2c.readfrom_mem(0x42, 0x10, 2)     # read 2 bytes from slave 0x42, slave memory 0x10
+    i2c.writeto_mem(0x42, 0x10, 'xy')   # write 2 bytes to slave 0x42, slave memory 0x10
+
+Note: for legacy I2C support see :ref:`pyb.I2C <pyb.I2C>`.
 
 CAN bus (controller area network)
 ---------------------------------

docs/pyboard/tutorial/repl.rst

@@ -96,8 +96,8 @@ If something goes wrong, you can reset the board in two ways. The first is to pr
 at the MicroPython prompt, which performs a soft reset.  You will see a message something like ::
 
     >>> 
-    PYB: sync filesystems
-    PYB: soft reboot
+    MPY: sync filesystems
+    MPY: soft reboot
     Micro Python v1.0 on 2014-05-03; PYBv1.0 with STM32F405RG
     Type "help()" for more information.
     >>>

docs/reference/repl.rst

@@ -174,8 +174,8 @@ variables no longer exist:
 
 .. code-block:: python
 
-    PYB: sync filesystems
-    PYB: soft reboot
+    MPY: sync filesystems
+    MPY: soft reboot
     MicroPython v1.5-51-g6f70283-dirty on 2015-10-30; PYBv1.0 with STM32F405RG
     Type "help()" for more information.
     >>> dir()

docs/reference/speed_python.rst

@@ -1,3 +1,5 @@
+.. _speed_python:
+
 Maximising MicroPython Speed
 ============================
 

docs/wipy/general.rst

@@ -178,7 +178,7 @@ Details on sleep modes
 * ``machine.idle()``: Power consumption: ~12mA (in WLAN STA mode). Wake sources:
   any hardware interrupt (including systick with period of 1ms), no special
   configuration required.
-* ``machine.sleep()``: 950uA (in WLAN STA mode). Wake sources are ``Pin``, ``RTC``
+* ``machine.lightsleep()``: 950uA (in WLAN STA mode). Wake sources are ``Pin``, ``RTC``
   and ``WLAN``
 * ``machine.deepsleep()``: ~350uA. Wake sources are ``Pin`` and ``RTC``.
 

docs/wipy/quickref.rst

@@ -28,7 +28,7 @@ See the :mod:`machine` module::
     machine.unique_id() # return the 6-byte unique id of the board (the WiPy's MAC address)
 
     machine.idle()        # average current decreases to (~12mA), any interrupts wake it up
-    machine.sleep()       # everything except for WLAN is powered down (~950uA avg. current)
+    machine.lightsleep()  # everything except for WLAN is powered down (~950uA avg. current)
                           # wakes from Pin, RTC or WLAN
     machine.deepsleep()   # deepest sleep mode, MCU starts from reset. Wakes from Pin and RTC.
 
@@ -163,7 +163,7 @@ See :ref:`machine.RTC <machine.RTC>` ::
     rtc_i = rtc.irq(trigger=RTC.ALARM0, handler=alarm_handler, wake=machine.SLEEP)
 
     # go into suspended mode waiting for the RTC alarm to expire and wake us up
-    machine.sleep()
+    machine.lightsleep()
 
 SD card
 -------
@@ -199,7 +199,7 @@ See :ref:`network.WLAN <network.WLAN>` and :mod:`machine`. ::
     # enable wake on WLAN
     wlan.irq(trigger=WLAN.ANY_EVENT, wake=machine.SLEEP)
     # go to sleep
-    machine.sleep()
+    machine.lightsleep()
     # now, connect to the FTP or the Telnet server and the WiPy will wake-up
 
 Telnet and FTP server

docs/wipy/tutorial/repl.rst

@@ -120,7 +120,7 @@ If something goes wrong, you can reset the board in two ways. The first is to pr
 at the MicroPython prompt, which performs a soft reset.  You will see a message something like::
 
     >>> 
-    PYB: soft reboot
+    MPY: soft reboot
     MicroPython v1.4.6-146-g1d8b5e5 on 2015-10-21; WiPy with CC3200
     Type "help()" for more information.
     >>>

drivers/display/ssd1306.py

@@ -96,6 +96,7 @@ class SSD1306_I2C(SSD1306):
         self.i2c = i2c
         self.addr = addr
         self.temp = bytearray(2)
+        self.write_list = [b'\x40', None] # Co=0, D/C#=1
         super().__init__(width, height, external_vcc)
 
     def write_cmd(self, cmd):
@@ -104,12 +105,8 @@ class SSD1306_I2C(SSD1306):
         self.i2c.writeto(self.addr, self.temp)
 
     def write_data(self, buf):
-        self.temp[0] = self.addr << 1
-        self.temp[1] = 0x40 # Co=0, D/C#=1
-        self.i2c.start()
-        self.i2c.write(self.temp)
-        self.i2c.write(buf)
-        self.i2c.stop()
+        self.write_list[1] = buf
+        self.i2c.writevto(self.addr, self.write_list)
 
 
 class SSD1306_SPI(SSD1306):

drivers/memory/spiflash.c

@@ -128,19 +128,14 @@ STATIC void mp_spiflash_write_cmd_addr(mp_spiflash_t *self, uint8_t cmd, uint32_
 
 STATIC int mp_spiflash_wait_sr(mp_spiflash_t *self, uint8_t mask, uint8_t val, uint32_t timeout) {
     uint8_t sr;
-    for (; timeout; --timeout) {
+    do {
         sr = mp_spiflash_read_cmd(self, CMD_RDSR, 1);
         if ((sr & mask) == val) {
-            break;
+            return 0; // success
         }
-    }
-    if ((sr & mask) == val) {
-        return 0; // success
-    } else if (timeout == 0) {
-        return -MP_ETIMEDOUT;
-    } else {
-        return -MP_EIO;
-    }
+    } while (timeout--);
+
+    return -MP_ETIMEDOUT;
 }
 
 STATIC int mp_spiflash_wait_wel1(mp_spiflash_t *self) {

extmod/machine_i2c.c

@@ -180,9 +180,9 @@ STATIC int mp_hal_i2c_read_byte(machine_i2c_obj_t *self, uint8_t *val, int nack)
 }
 
 // return value:
-//  >=0 - number of acks received
+//  >=0 - success; for read it's 0, for write it's number of acks received
 //   <0 - error, with errno being the negative of the return value
-int mp_machine_soft_i2c_writeto(mp_obj_base_t *self_in, uint16_t addr, const uint8_t *src, size_t len, bool stop) {
+int mp_machine_soft_i2c_transfer(mp_obj_base_t *self_in, uint16_t addr, size_t n, mp_machine_i2c_buf_t *bufs, unsigned int flags) {
     machine_i2c_obj_t *self = (machine_i2c_obj_t*)self_in;
 
     // start the I2C transaction
@@ -192,7 +192,7 @@ int mp_machine_soft_i2c_writeto(mp_obj_base_t *self_in, uint16_t addr, const uin
     }
 
     // write the slave address
-    ret = mp_hal_i2c_write_byte(self, addr << 1);
+    ret = mp_hal_i2c_write_byte(self, (addr << 1) | (flags & MP_MACHINE_I2C_FLAG_READ));
     if (ret < 0) {
         return ret;
     } else if (ret != 0) {
@@ -201,69 +201,102 @@ int mp_machine_soft_i2c_writeto(mp_obj_base_t *self_in, uint16_t addr, const uin
         return -MP_ENODEV;
     }
 
-    // write the buffer to the I2C memory
-    int num_acks = 0;
-    while (len--) {
-        ret = mp_hal_i2c_write_byte(self, *src++);
-        if (ret < 0) {
-            return ret;
-        } else if (ret != 0) {
-            // nack received, stop sending
-            break;
+    int transfer_ret = 0;
+    for (; n--; ++bufs) {
+        size_t len = bufs->len;
+        uint8_t *buf = bufs->buf;
+        if (flags & MP_MACHINE_I2C_FLAG_READ) {
+            // read bytes from the slave into the given buffer(s)
+            while (len--) {
+                ret = mp_hal_i2c_read_byte(self, buf++, (n | len) == 0);
+                if (ret != 0) {
+                    return ret;
+                }
+            }
+        } else {
+            // write bytes from the given buffer(s) to the slave
+            while (len--) {
+                ret = mp_hal_i2c_write_byte(self, *buf++);
+                if (ret < 0) {
+                    return ret;
+                } else if (ret != 0) {
+                    // nack received, stop sending
+                    n = 0;
+                    break;
+                }
+                ++transfer_ret; // count the number of acks
+            }
         }
-        ++num_acks;
     }
 
     // finish the I2C transaction
-    if (stop) {
+    if (flags & MP_MACHINE_I2C_FLAG_STOP) {
         ret = mp_hal_i2c_stop(self);
         if (ret != 0) {
             return ret;
         }
     }
 
-    return num_acks;
+    return transfer_ret;
 }
 
-// return value:
-//    0 - success
-//   <0 - error, with errno being the negative of the return value
-int mp_machine_soft_i2c_readfrom(mp_obj_base_t *self_in, uint16_t addr, uint8_t *dest, size_t len, bool stop) {
-    machine_i2c_obj_t *self = (machine_i2c_obj_t*)self_in;
+/******************************************************************************/
+// Generic helper functions
 
-    // start the I2C transaction
-    int ret = mp_hal_i2c_start(self);
-    if (ret != 0) {
-        return ret;
-    }
-
-    // write the slave address
-    ret = mp_hal_i2c_write_byte(self, (addr << 1) | 1);
-    if (ret < 0) {
-        return ret;
-    } else if (ret != 0) {
-        // nack received, release the bus cleanly
-        mp_hal_i2c_stop(self);
-        return -MP_ENODEV;
-    }
-
-    // read the bytes from the slave
-    while (len--) {
-        ret = mp_hal_i2c_read_byte(self, dest++, len == 0);
-        if (ret != 0) {
-            return ret;
+// For use by ports that require a single buffer of data for a read/write transfer
+int mp_machine_i2c_transfer_adaptor(mp_obj_base_t *self, uint16_t addr, size_t n, mp_machine_i2c_buf_t *bufs, unsigned int flags) {
+    size_t len;
+    uint8_t *buf;
+    if (n == 1) {
+        // Use given single buffer
+        len = bufs[0].len;
+        buf = bufs[0].buf;
+    } else {
+        // Combine buffers into a single one
+        len = 0;
+        for (size_t i = 0; i < n; ++i) {
+            len += bufs[i].len;
+        }
+        buf = m_new(uint8_t, len);
+        if (!(flags & MP_MACHINE_I2C_FLAG_READ)) {
+            len = 0;
+            for (size_t i = 0; i < n; ++i) {
+                memcpy(buf + len, bufs[i].buf, bufs[i].len);
+                len += bufs[i].len;
+            }
         }
     }
 
-    // finish the I2C transaction
-    if (stop) {
-        ret = mp_hal_i2c_stop(self);
-        if (ret != 0) {
-            return ret;
+    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
+    int ret = i2c_p->transfer_single(self, addr, len, buf, flags);
+
+    if (n > 1) {
+        if (flags & MP_MACHINE_I2C_FLAG_READ) {
+            // Copy data from single buffer to individual ones
+            len = 0;
+            for (size_t i = 0; i < n; ++i) {
+                memcpy(bufs[i].buf, buf + len, bufs[i].len);
+                len += bufs[i].len;
+            }
         }
+        m_del(uint8_t, buf, len);
     }
 
-    return 0; // success
+    return ret;
+}
+
+STATIC int mp_machine_i2c_readfrom(mp_obj_base_t *self, uint16_t addr, uint8_t *dest, size_t len, bool stop) {
+    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
+    mp_machine_i2c_buf_t buf = {.len = len, .buf = dest};
+    unsigned int flags = MP_MACHINE_I2C_FLAG_READ | (stop ? MP_MACHINE_I2C_FLAG_STOP : 0);
+    return i2c_p->transfer(self, addr, 1, &buf, flags);
+}
+
+STATIC int mp_machine_i2c_writeto(mp_obj_base_t *self, uint16_t addr, const uint8_t *src, size_t len, bool stop) {
+    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
+    mp_machine_i2c_buf_t buf = {.len = len, .buf = (uint8_t*)src};
+    unsigned int flags = stop ? MP_MACHINE_I2C_FLAG_STOP : 0;
+    return i2c_p->transfer(self, addr, 1, &buf, flags);
 }
 
 /******************************************************************************/
@@ -318,11 +351,10 @@ MP_DEFINE_CONST_FUN_OBJ_KW(machine_i2c_init_obj, 1, machine_i2c_obj_init);
 
 STATIC mp_obj_t machine_i2c_scan(mp_obj_t self_in) {
     mp_obj_base_t *self = MP_OBJ_TO_PTR(self_in);
-    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
     mp_obj_t list = mp_obj_new_list(0, NULL);
     // 7-bit addresses 0b0000xxx and 0b1111xxx are reserved
     for (int addr = 0x08; addr < 0x78; ++addr) {
-        int ret = i2c_p->writeto(self, addr, NULL, 0, true);
+        int ret = mp_machine_i2c_writeto(self, addr, NULL, 0, true);
         if (ret == 0) {
             mp_obj_list_append(list, MP_OBJ_NEW_SMALL_INT(addr));
         }
@@ -407,12 +439,11 @@ MP_DEFINE_CONST_FUN_OBJ_2(machine_i2c_write_obj, machine_i2c_write);
 
 STATIC mp_obj_t machine_i2c_readfrom(size_t n_args, const mp_obj_t *args) {
     mp_obj_base_t *self = (mp_obj_base_t*)MP_OBJ_TO_PTR(args[0]);
-    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
     mp_int_t addr = mp_obj_get_int(args[1]);
     vstr_t vstr;
     vstr_init_len(&vstr, mp_obj_get_int(args[2]));
     bool stop = (n_args == 3) ? true : mp_obj_is_true(args[3]);
-    int ret = i2c_p->readfrom(self, addr, (uint8_t*)vstr.buf, vstr.len, stop);
+    int ret = mp_machine_i2c_readfrom(self, addr, (uint8_t*)vstr.buf, vstr.len, stop);
     if (ret < 0) {
         mp_raise_OSError(-ret);
     }
@@ -422,12 +453,11 @@ MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(machine_i2c_readfrom_obj, 3, 4, machine_i2c_
 
 STATIC mp_obj_t machine_i2c_readfrom_into(size_t n_args, const mp_obj_t *args) {
     mp_obj_base_t *self = (mp_obj_base_t*)MP_OBJ_TO_PTR(args[0]);
-    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
     mp_int_t addr = mp_obj_get_int(args[1]);
     mp_buffer_info_t bufinfo;
     mp_get_buffer_raise(args[2], &bufinfo, MP_BUFFER_WRITE);
     bool stop = (n_args == 3) ? true : mp_obj_is_true(args[3]);
-    int ret = i2c_p->readfrom(self, addr, bufinfo.buf, bufinfo.len, stop);
+    int ret = mp_machine_i2c_readfrom(self, addr, bufinfo.buf, bufinfo.len, stop);
     if (ret < 0) {
         mp_raise_OSError(-ret);
     }
@@ -437,12 +467,11 @@ MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(machine_i2c_readfrom_into_obj, 3, 4, machine
 
 STATIC mp_obj_t machine_i2c_writeto(size_t n_args, const mp_obj_t *args) {
     mp_obj_base_t *self = (mp_obj_base_t*)MP_OBJ_TO_PTR(args[0]);
-    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
     mp_int_t addr = mp_obj_get_int(args[1]);
     mp_buffer_info_t bufinfo;
     mp_get_buffer_raise(args[2], &bufinfo, MP_BUFFER_READ);
     bool stop = (n_args == 3) ? true : mp_obj_is_true(args[3]);
-    int ret = i2c_p->writeto(self, addr,  bufinfo.buf, bufinfo.len, stop);
+    int ret = mp_machine_i2c_writeto(self, addr, bufinfo.buf, bufinfo.len, stop);
     if (ret < 0) {
         mp_raise_OSError(-ret);
     }
@@ -451,53 +480,87 @@ STATIC mp_obj_t machine_i2c_writeto(size_t n_args, const mp_obj_t *args) {
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(machine_i2c_writeto_obj, 3, 4, machine_i2c_writeto);
 
+STATIC mp_obj_t machine_i2c_writevto(size_t n_args, const mp_obj_t *args) {
+    mp_obj_base_t *self = (mp_obj_base_t*)MP_OBJ_TO_PTR(args[0]);
+    mp_int_t addr = mp_obj_get_int(args[1]);
+
+    // Get the list of data buffer(s) to write
+    size_t nitems;
+    const mp_obj_t *items;
+    mp_obj_get_array(args[2], &nitems, (mp_obj_t**)&items);
+
+    // Get the stop argument
+    bool stop = (n_args == 3) ? true : mp_obj_is_true(args[3]);
+
+    // Extract all buffer data, skipping zero-length buffers
+    size_t alloc = nitems == 0 ? 1 : nitems;
+    size_t nbufs = 0;
+    mp_machine_i2c_buf_t *bufs = mp_local_alloc(alloc * sizeof(mp_machine_i2c_buf_t));
+    for (; nitems--; ++items) {
+        mp_buffer_info_t bufinfo;
+        mp_get_buffer_raise(*items, &bufinfo, MP_BUFFER_READ);
+        if (bufinfo.len > 0) {
+            bufs[nbufs].len = bufinfo.len;
+            bufs[nbufs++].buf = bufinfo.buf;
+        }
+    }
+
+    // Make sure there is at least one buffer, empty if needed
+    if (nbufs == 0) {
+        bufs[0].len = 0;
+        bufs[0].buf = NULL;
+        nbufs = 1;
+    }
+
+    // Do the I2C transfer
+    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
+    int ret = i2c_p->transfer(self, addr, nbufs, bufs, stop ? MP_MACHINE_I2C_FLAG_STOP : 0);
+    mp_local_free(bufs);
+
+    if (ret < 0) {
+        mp_raise_OSError(-ret);
+    }
+
+    // Return number of acks received
+    return MP_OBJ_NEW_SMALL_INT(ret);
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(machine_i2c_writevto_obj, 3, 4, machine_i2c_writevto);
+
 STATIC int read_mem(mp_obj_t self_in, uint16_t addr, uint32_t memaddr, uint8_t addrsize, uint8_t *buf, size_t len) {
     mp_obj_base_t *self = (mp_obj_base_t*)MP_OBJ_TO_PTR(self_in);
-    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
     uint8_t memaddr_buf[4];
     size_t memaddr_len = 0;
     for (int16_t i = addrsize - 8; i >= 0; i -= 8) {
         memaddr_buf[memaddr_len++] = memaddr >> i;
     }
-    int ret = i2c_p->writeto(self, addr, memaddr_buf, memaddr_len, false);
+    int ret = mp_machine_i2c_writeto(self, addr, memaddr_buf, memaddr_len, false);
     if (ret != memaddr_len) {
         // must generate STOP
-        i2c_p->writeto(self, addr, NULL, 0, true);
+        mp_machine_i2c_writeto(self, addr, NULL, 0, true);
         return ret;
     }
-    return i2c_p->readfrom(self, addr, buf, len, true);
+    return mp_machine_i2c_readfrom(self, addr, buf, len, true);
 }
 
-#define MAX_MEMADDR_SIZE (4)
-#define BUF_STACK_SIZE (12)
-
 STATIC int write_mem(mp_obj_t self_in, uint16_t addr, uint32_t memaddr, uint8_t addrsize, const uint8_t *buf, size_t len) {
     mp_obj_base_t *self = (mp_obj_base_t*)MP_OBJ_TO_PTR(self_in);
-    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
 
-    // need some memory to create the buffer to send; try to use stack if possible
-    uint8_t buf2_stack[MAX_MEMADDR_SIZE + BUF_STACK_SIZE];
-    uint8_t *buf2;
-    size_t buf2_alloc = 0;
-    if (len <= BUF_STACK_SIZE) {
-        buf2 = buf2_stack;
-    } else {
-        buf2_alloc = MAX_MEMADDR_SIZE + len;
-        buf2 = m_new(uint8_t, buf2_alloc);
-    }
-
-    // create the buffer to send
+    // Create buffer with memory address
     size_t memaddr_len = 0;
+    uint8_t memaddr_buf[4];
     for (int16_t i = addrsize - 8; i >= 0; i -= 8) {
-        buf2[memaddr_len++] = memaddr >> i;
+        memaddr_buf[memaddr_len++] = memaddr >> i;
     }
-    memcpy(buf2 + memaddr_len, buf, len);
 
-    int ret = i2c_p->writeto(self, addr, buf2, memaddr_len + len, true);
-    if (buf2_alloc != 0) {
-        m_del(uint8_t, buf2, buf2_alloc);
-    }
-    return ret;
+    // Create partial write buffers
+    mp_machine_i2c_buf_t bufs[2] = {
+        {.len = memaddr_len, .buf = memaddr_buf},
+        {.len = len, .buf = (uint8_t*)buf},
+    };
+
+    // Do I2C transfer
+    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
+    return i2c_p->transfer(self, addr, 2, bufs, MP_MACHINE_I2C_FLAG_STOP);
 }
 
 STATIC const mp_arg_t machine_i2c_mem_allowed_args[] = {
@@ -584,6 +647,7 @@ STATIC const mp_rom_map_elem_t machine_i2c_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_readfrom), MP_ROM_PTR(&machine_i2c_readfrom_obj) },
     { MP_ROM_QSTR(MP_QSTR_readfrom_into), MP_ROM_PTR(&machine_i2c_readfrom_into_obj) },
     { MP_ROM_QSTR(MP_QSTR_writeto), MP_ROM_PTR(&machine_i2c_writeto_obj) },
+    { MP_ROM_QSTR(MP_QSTR_writevto), MP_ROM_PTR(&machine_i2c_writevto_obj) },
 
     // memory operations
     { MP_ROM_QSTR(MP_QSTR_readfrom_mem), MP_ROM_PTR(&machine_i2c_readfrom_mem_obj) },
@@ -625,8 +689,7 @@ STATIC const mp_machine_i2c_p_t mp_machine_soft_i2c_p = {
     .stop = (int(*)(mp_obj_base_t*))mp_hal_i2c_stop,
     .read = mp_machine_soft_i2c_read,
     .write = mp_machine_soft_i2c_write,
-    .readfrom = mp_machine_soft_i2c_readfrom,
-    .writeto = mp_machine_soft_i2c_writeto,
+    .transfer = mp_machine_soft_i2c_transfer,
 };
 
 const mp_obj_type_t machine_i2c_type = {

extmod/machine_i2c.h

@@ -28,15 +28,24 @@
 
 #include "py/obj.h"
 
+#define MP_MACHINE_I2C_FLAG_READ (0x01) // if not set then it's a write
+#define MP_MACHINE_I2C_FLAG_STOP (0x02)
+
+typedef struct _mp_machine_i2c_buf_t {
+    size_t len;
+    uint8_t *buf;
+} mp_machine_i2c_buf_t;
+
 // I2C protocol
 // the first 4 methods can be NULL, meaning operation is not supported
+// transfer_single only needs to be set if transfer=mp_machine_i2c_transfer_adaptor
 typedef struct _mp_machine_i2c_p_t {
     int (*start)(mp_obj_base_t *obj);
     int (*stop)(mp_obj_base_t *obj);
     int (*read)(mp_obj_base_t *obj, uint8_t *dest, size_t len, bool nack);
     int (*write)(mp_obj_base_t *obj, const uint8_t *src, size_t len);
-    int (*readfrom)(mp_obj_base_t *obj, uint16_t addr, uint8_t *dest, size_t len, bool stop);
-    int (*writeto)(mp_obj_base_t *obj, uint16_t addr, const uint8_t *src, size_t len, bool stop);
+    int (*transfer)(mp_obj_base_t *obj, uint16_t addr, size_t n, mp_machine_i2c_buf_t *bufs, unsigned int flags);
+    int (*transfer_single)(mp_obj_base_t *obj, uint16_t addr, size_t len, uint8_t *buf, unsigned int flags);
 } mp_machine_i2c_p_t;
 
 typedef struct _mp_machine_soft_i2c_obj_t {
@@ -50,7 +59,7 @@ typedef struct _mp_machine_soft_i2c_obj_t {
 extern const mp_obj_type_t machine_i2c_type;
 extern const mp_obj_dict_t mp_machine_soft_i2c_locals_dict;
 
-int mp_machine_soft_i2c_readfrom(mp_obj_base_t *self_in, uint16_t addr, uint8_t *dest, size_t len, bool stop);
-int mp_machine_soft_i2c_writeto(mp_obj_base_t *self_in, uint16_t addr, const uint8_t *src, size_t len, bool stop);
+int mp_machine_i2c_transfer_adaptor(mp_obj_base_t *self, uint16_t addr, size_t n, mp_machine_i2c_buf_t *bufs, unsigned int flags);
+int mp_machine_soft_i2c_transfer(mp_obj_base_t *self, uint16_t addr, size_t n, mp_machine_i2c_buf_t *bufs, unsigned int flags);
 
 #endif // MICROPY_INCLUDED_EXTMOD_MACHINE_I2C_H

extmod/machine_signal.c

@@ -43,13 +43,13 @@ typedef struct _machine_signal_t {
 } machine_signal_t;
 
 STATIC mp_obj_t signal_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) {
-    mp_obj_t pin = args[0];
+    mp_obj_t pin;
     bool invert = false;
 
     #if defined(MICROPY_PY_MACHINE_PIN_MAKE_NEW)
     mp_pin_p_t *pin_p = NULL;
 
-    if (MP_OBJ_IS_OBJ(pin)) {
+    if (n_args > 0 && mp_obj_is_obj(args[0])) {
         mp_obj_base_t *pin_base = (mp_obj_base_t*)MP_OBJ_TO_PTR(args[0]);
         pin_p = (mp_pin_p_t*)pin_base->type->protocol;
     }
@@ -96,6 +96,7 @@ STATIC mp_obj_t signal_make_new(const mp_obj_type_t *type, size_t n_args, size_t
     // Otherwise there should be 1 or 2 args
     {
         if (n_args == 1) {
+            pin = args[0];
             if (n_kw == 0) {
             } else if (n_kw == 1 && args[1] == MP_OBJ_NEW_QSTR(MP_QSTR_invert)) {
                 invert = mp_obj_is_true(args[2]);

extmod/misc.h

@@ -35,6 +35,7 @@
 MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_uos_dupterm_obj);
 
 #if MICROPY_PY_OS_DUPTERM
+bool mp_uos_dupterm_is_builtin_stream(mp_const_obj_t stream);
 int mp_uos_dupterm_rx_chr(void);
 void mp_uos_dupterm_tx_strn(const char *str, size_t len);
 void mp_uos_deactivate(size_t dupterm_idx, const char *msg, mp_obj_t exc);

extmod/modlwip.c

@@ -3,7 +3,7 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2013, 2014 Damien P. George
+ * Copyright (c) 2013-2019 Damien P. George
  * Copyright (c) 2015 Galen Hazelwood
  * Copyright (c) 2015-2017 Paul Sokolovsky
  *
@@ -57,6 +57,10 @@
 #define DEBUG_printf(...) (void)0
 #endif
 
+// Timeout between closing a TCP socket and doing a tcp_abort on that
+// socket, if the connection isn't closed cleanly in that time.
+#define MICROPY_PY_LWIP_TCP_CLOSE_TIMEOUT_MS (10000)
+
 // All socket options should be globally distinct,
 // because we ignore option levels for efficiency.
 #define IP_ADD_MEMBERSHIP 0x400
@@ -69,6 +73,13 @@
 #define ip_reset_option(pcb, opt) ((pcb)->so_options &= ~(opt))
 #endif
 
+// A port can define these hooks to provide concurrency protection
+#ifndef MICROPY_PY_LWIP_ENTER
+#define MICROPY_PY_LWIP_ENTER
+#define MICROPY_PY_LWIP_REENTER
+#define MICROPY_PY_LWIP_EXIT
+#endif
+
 #ifdef MICROPY_PY_LWIP_SLIP
 #include "netif/slipif.h"
 #include "lwip/sio.h"
@@ -308,6 +319,38 @@ static inline void poll_sockets(void) {
 #endif
 }
 
+STATIC struct tcp_pcb *volatile *lwip_socket_incoming_array(lwip_socket_obj_t *socket) {
+    if (socket->incoming.connection.alloc == 0) {
+        return &socket->incoming.connection.tcp.item;
+    } else {
+        return &socket->incoming.connection.tcp.array[0];
+    }
+}
+
+STATIC void lwip_socket_free_incoming(lwip_socket_obj_t *socket) {
+    bool socket_is_listener =
+        socket->type == MOD_NETWORK_SOCK_STREAM
+        && socket->pcb.tcp->state == LISTEN;
+
+    if (!socket_is_listener) {
+        if (socket->incoming.pbuf != NULL) {
+            pbuf_free(socket->incoming.pbuf);
+            socket->incoming.pbuf = NULL;
+        }
+    } else {
+        uint8_t alloc = socket->incoming.connection.alloc;
+        struct tcp_pcb *volatile *tcp_array = lwip_socket_incoming_array(socket);
+        for (uint8_t i = 0; i < alloc; ++i) {
+            // Deregister callback and abort
+            if (tcp_array[i] != NULL) {
+                tcp_poll(tcp_array[i], NULL, 0);
+                tcp_abort(tcp_array[i]);
+                tcp_array[i] = NULL;
+            }
+        }
+    }
+}
+
 /*******************************************************************************/
 // Callback functions for the lwIP raw API.
 
@@ -341,6 +384,8 @@ STATIC void _lwip_udp_incoming(void *arg, struct udp_pcb *upcb, struct pbuf *p,
 STATIC void _lwip_tcp_error(void *arg, err_t err) {
     lwip_socket_obj_t *socket = (lwip_socket_obj_t*)arg;
 
+    // Free any incoming buffers or connections that are stored
+    lwip_socket_free_incoming(socket);
     // Pass the error code back via the connection variable.
     socket->state = err;
     // If we got here, the lwIP stack either has deallocated or will deallocate the pcb.
@@ -355,6 +400,43 @@ STATIC err_t _lwip_tcp_connected(void *arg, struct tcp_pcb *tpcb, err_t err) {
     return ERR_OK;
 }
 
+// Handle errors (eg connection aborted) on TCP PCBs that have been put on the
+// accept queue but are not yet actually accepted.
+STATIC void _lwip_tcp_err_unaccepted(void *arg, err_t err) {
+    struct tcp_pcb *pcb = (struct tcp_pcb*)arg;
+
+    // The ->connected entry is repurposed to store the parent socket; this is safe
+    // because it's only ever used by lwIP if tcp_connect is called on the TCP PCB.
+    lwip_socket_obj_t *socket = (lwip_socket_obj_t*)pcb->connected;
+
+    // Array is not volatile because thiss callback is executed within the lwIP context
+    uint8_t alloc = socket->incoming.connection.alloc;
+    struct tcp_pcb **tcp_array = (struct tcp_pcb**)lwip_socket_incoming_array(socket);
+
+    // Search for PCB on the accept queue of the parent socket
+    struct tcp_pcb **shift_down = NULL;
+    uint8_t i = socket->incoming.connection.iget;
+    do {
+        if (shift_down == NULL) {
+            if (tcp_array[i] == pcb) {
+                shift_down = &tcp_array[i];
+            }
+        } else {
+            *shift_down = tcp_array[i];
+            shift_down = &tcp_array[i];
+        }
+        if (++i >= alloc) {
+            i = 0;
+        }
+    } while (i != socket->incoming.connection.iput);
+
+    // PCB found in queue, remove it
+    if (shift_down != NULL) {
+        *shift_down = NULL;
+        socket->incoming.connection.iput = shift_down - tcp_array;
+    }
+}
+
 // By default, a child socket of listen socket is created with recv
 // handler which discards incoming pbuf's. We don't want to do that,
 // so set this handler which requests lwIP to keep pbuf's and deliver
@@ -369,7 +451,8 @@ STATIC err_t _lwip_tcp_recv_unaccepted(void *arg, struct tcp_pcb *pcb, struct pb
 // from accept callback itself.
 STATIC err_t _lwip_tcp_accept_finished(void *arg, struct tcp_pcb *pcb)
 {
-    lwip_socket_obj_t *socket = (lwip_socket_obj_t*)arg;
+    // The ->connected entry of the pcb holds the listening socket of the accept
+    lwip_socket_obj_t *socket = (lwip_socket_obj_t*)pcb->connected;
     tcp_poll(pcb, NULL, 0);
     exec_user_callback(socket);
     return ERR_OK;
@@ -377,19 +460,19 @@ STATIC err_t _lwip_tcp_accept_finished(void *arg, struct tcp_pcb *pcb)
 
 // Callback for incoming tcp connections.
 STATIC err_t _lwip_tcp_accept(void *arg, struct tcp_pcb *newpcb, err_t err) {
+    // err can be ERR_MEM to notify us that there was no memory for an incoming connection
+    if (err != ERR_OK) {
+        return ERR_OK;
+    }
+
     lwip_socket_obj_t *socket = (lwip_socket_obj_t*)arg;
     tcp_recv(newpcb, _lwip_tcp_recv_unaccepted);
 
     // Search for an empty slot to store the new connection
-    struct tcp_pcb *volatile *tcp_array;
-    if (socket->incoming.connection.alloc == 0) {
-        tcp_array = &socket->incoming.connection.tcp.item;
-    } else {
-        tcp_array = socket->incoming.connection.tcp.array;
-    }
-    if (tcp_array[socket->incoming.connection.iput] == NULL) {
+    struct tcp_pcb *volatile *slot = &lwip_socket_incoming_array(socket)[socket->incoming.connection.iput];
+    if (*slot == NULL) {
         // Have an empty slot to store waiting connection
-        tcp_array[socket->incoming.connection.iput] = newpcb;
+        *slot = newpcb;
         if (++socket->incoming.connection.iput >= socket->incoming.connection.alloc) {
             socket->incoming.connection.iput = 0;
         }
@@ -399,6 +482,15 @@ STATIC err_t _lwip_tcp_accept(void *arg, struct tcp_pcb *newpcb, err_t err) {
             // is idle.
             tcp_poll(newpcb, _lwip_tcp_accept_finished, 1);
         }
+
+        // Set the error callback to handle the case of a dropped connection before we
+        // have a chance to take it off the accept queue.
+        // The ->connected entry is repurposed to store the parent socket; this is safe
+        // because it's only ever used by lwIP if tcp_connect is called on the TCP PCB.
+        newpcb->connected = (void*)socket;
+        tcp_arg(newpcb, newpcb);
+        tcp_err(newpcb, _lwip_tcp_err_unaccepted);
+
         return ERR_OK;
     }
 
@@ -444,9 +536,12 @@ STATIC mp_uint_t lwip_udp_send(lwip_socket_obj_t *socket, const byte *buf, mp_ui
         len = 0xffff;
     }
 
+    MICROPY_PY_LWIP_ENTER
+
     // FIXME: maybe PBUF_ROM?
     struct pbuf *p = pbuf_alloc(PBUF_TRANSPORT, len, PBUF_RAM);
     if (p == NULL) {
+        MICROPY_PY_LWIP_EXIT
         *_errno = MP_ENOMEM;
         return -1;
     }
@@ -464,6 +559,8 @@ STATIC mp_uint_t lwip_udp_send(lwip_socket_obj_t *socket, const byte *buf, mp_ui
 
     pbuf_free(p);
 
+    MICROPY_PY_LWIP_EXIT
+
     // udp_sendto can return 1 on occasion for ESP8266 port.  It's not known why
     // but it seems that the send actually goes through without error in this case.
     // So we treat such cases as a success until further investigation.
@@ -502,10 +599,14 @@ STATIC mp_uint_t lwip_udp_receive(lwip_socket_obj_t *socket, byte *buf, mp_uint_
 
     struct pbuf *p = socket->incoming.pbuf;
 
+    MICROPY_PY_LWIP_ENTER
+
     u16_t result = pbuf_copy_partial(p, buf, ((p->tot_len > len) ? len : p->tot_len), 0);
     pbuf_free(p);
     socket->incoming.pbuf = NULL;
 
+    MICROPY_PY_LWIP_EXIT
+
     return (mp_uint_t) result;
 }
 
@@ -517,17 +618,29 @@ STATIC mp_uint_t lwip_udp_receive(lwip_socket_obj_t *socket, byte *buf, mp_uint_
         } \
         assert(socket->pcb.tcp);
 
+// Version of above for use when lock is held
+#define STREAM_ERROR_CHECK_WITH_LOCK(socket) \
+        if (socket->state < 0) { \
+            *_errno = error_lookup_table[-socket->state]; \
+            MICROPY_PY_LWIP_EXIT \
+            return MP_STREAM_ERROR; \
+        } \
+        assert(socket->pcb.tcp);
+
 
 // Helper function for send/sendto to handle TCP packets
 STATIC mp_uint_t lwip_tcp_send(lwip_socket_obj_t *socket, const byte *buf, mp_uint_t len, int *_errno) {
     // Check for any pending errors
     STREAM_ERROR_CHECK(socket);
 
+    MICROPY_PY_LWIP_ENTER
+
     u16_t available = tcp_sndbuf(socket->pcb.tcp);
 
     if (available == 0) {
         // Non-blocking socket
         if (socket->timeout == 0) {
+            MICROPY_PY_LWIP_EXIT
             *_errno = MP_EAGAIN;
             return MP_STREAM_ERROR;
         }
@@ -540,15 +653,17 @@ STATIC mp_uint_t lwip_tcp_send(lwip_socket_obj_t *socket, const byte *buf, mp_ui
         // reset) by error callback.
         // Avoid sending too small packets, so wait until at least 16 bytes available
         while (socket->state >= STATE_CONNECTED && (available = tcp_sndbuf(socket->pcb.tcp)) < 16) {
+            MICROPY_PY_LWIP_EXIT
             if (socket->timeout != -1 && mp_hal_ticks_ms() - start > socket->timeout) {
                 *_errno = MP_ETIMEDOUT;
                 return MP_STREAM_ERROR;
             }
             poll_sockets();
+            MICROPY_PY_LWIP_REENTER
         }
 
         // While we waited, something could happen
-        STREAM_ERROR_CHECK(socket);
+        STREAM_ERROR_CHECK_WITH_LOCK(socket);
     }
 
     u16_t write_len = MIN(available, len);
@@ -560,6 +675,8 @@ STATIC mp_uint_t lwip_tcp_send(lwip_socket_obj_t *socket, const byte *buf, mp_ui
         err = tcp_output(socket->pcb.tcp);
     }
 
+    MICROPY_PY_LWIP_EXIT
+
     if (err != ERR_OK) {
         *_errno = error_lookup_table[-err];
         return MP_STREAM_ERROR;
@@ -605,6 +722,8 @@ STATIC mp_uint_t lwip_tcp_receive(lwip_socket_obj_t *socket, byte *buf, mp_uint_
         }
     }
 
+    MICROPY_PY_LWIP_ENTER
+
     assert(socket->pcb.tcp != NULL);
 
     struct pbuf *p = socket->incoming.pbuf;
@@ -630,6 +749,8 @@ STATIC mp_uint_t lwip_tcp_receive(lwip_socket_obj_t *socket, byte *buf, mp_uint_
     }
     tcp_recved(socket->pcb.tcp, len);
 
+    MICROPY_PY_LWIP_EXIT
+
     return len;
 }
 
@@ -769,47 +890,58 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_2(lwip_socket_listen_obj, lwip_socket_listen);
 STATIC mp_obj_t lwip_socket_accept(mp_obj_t self_in) {
     lwip_socket_obj_t *socket = MP_OBJ_TO_PTR(self_in);
 
-    if (socket->pcb.tcp == NULL) {
-        mp_raise_OSError(MP_EBADF);
-    }
     if (socket->type != MOD_NETWORK_SOCK_STREAM) {
         mp_raise_OSError(MP_EOPNOTSUPP);
     }
+
+    // Create new socket object, do it here because we must not raise an out-of-memory
+    // exception when the LWIP concurrency lock is held
+    lwip_socket_obj_t *socket2 = m_new_obj_with_finaliser(lwip_socket_obj_t);
+    socket2->base.type = &lwip_socket_type;
+
+    MICROPY_PY_LWIP_ENTER
+
+    if (socket->pcb.tcp == NULL) {
+        MICROPY_PY_LWIP_EXIT
+        m_del_obj(lwip_socket_obj_t, socket2);
+        mp_raise_OSError(MP_EBADF);
+    }
+
     // I need to do this because "tcp_accepted", later, is a macro.
     struct tcp_pcb *listener = socket->pcb.tcp;
     if (listener->state != LISTEN) {
+        MICROPY_PY_LWIP_EXIT
+        m_del_obj(lwip_socket_obj_t, socket2);
         mp_raise_OSError(MP_EINVAL);
     }
 
     // accept incoming connection
-    struct tcp_pcb *volatile *incoming_connection;
-    if (socket->incoming.connection.alloc == 0) {
-        incoming_connection = &socket->incoming.connection.tcp.item;
-    } else {
-        incoming_connection = &socket->incoming.connection.tcp.array[socket->incoming.connection.iget];
-    }
+    struct tcp_pcb *volatile *incoming_connection = &lwip_socket_incoming_array(socket)[socket->incoming.connection.iget];
     if (*incoming_connection == NULL) {
         if (socket->timeout == 0) {
+            MICROPY_PY_LWIP_EXIT
+            m_del_obj(lwip_socket_obj_t, socket2);
             mp_raise_OSError(MP_EAGAIN);
         } else if (socket->timeout != -1) {
-            for (mp_uint_t retries = socket->timeout / 100; retries--;) {
+            mp_uint_t retries = socket->timeout / 100;
+            while (*incoming_connection == NULL) {
+                MICROPY_PY_LWIP_EXIT
+                if (retries-- == 0) {
+                    m_del_obj(lwip_socket_obj_t, socket2);
+                    mp_raise_OSError(MP_ETIMEDOUT);
+                }
                 mp_hal_delay_ms(100);
-                if (*incoming_connection != NULL) break;
-            }
-            if (*incoming_connection == NULL) {
-                mp_raise_OSError(MP_ETIMEDOUT);
+                MICROPY_PY_LWIP_REENTER
             }
         } else {
             while (*incoming_connection == NULL) {
+                MICROPY_PY_LWIP_EXIT
                 poll_sockets();
+                MICROPY_PY_LWIP_REENTER
             }
         }
     }
 
-    // create new socket object
-    lwip_socket_obj_t *socket2 = m_new_obj_with_finaliser(lwip_socket_obj_t);
-    socket2->base.type = &lwip_socket_type;
-
     // We get a new pcb handle...
     socket2->pcb.tcp = *incoming_connection;
     if (++socket->incoming.connection.iget >= socket->incoming.connection.alloc) {
@@ -831,6 +963,8 @@ STATIC mp_obj_t lwip_socket_accept(mp_obj_t self_in) {
 
     tcp_accepted(listener);
 
+    MICROPY_PY_LWIP_EXIT
+
     // make the return value
     uint8_t ip[NETUTILS_IPV4ADDR_BUFSIZE];
     memcpy(ip, &(socket2->pcb.tcp->remote_ip), sizeof(ip));
@@ -867,16 +1001,21 @@ STATIC mp_obj_t lwip_socket_connect(mp_obj_t self_in, mp_obj_t addr_in) {
                     mp_raise_OSError(MP_EALREADY);
                 }
             }
+
             // Register our receive callback.
+            MICROPY_PY_LWIP_ENTER
             tcp_recv(socket->pcb.tcp, _lwip_tcp_recv);
             socket->state = STATE_CONNECTING;
             err = tcp_connect(socket->pcb.tcp, &dest, port, _lwip_tcp_connected);
             if (err != ERR_OK) {
+                MICROPY_PY_LWIP_EXIT
                 socket->state = STATE_NEW;
                 mp_raise_OSError(error_lookup_table[-err]);
             }
             socket->peer_port = (mp_uint_t)port;
             memcpy(socket->peer, &dest, sizeof(socket->peer));
+            MICROPY_PY_LWIP_EXIT
+
             // And now we wait...
             if (socket->timeout != -1) {
                 for (mp_uint_t retries = socket->timeout / 100; retries--;) {
@@ -1207,10 +1346,19 @@ STATIC mp_uint_t lwip_socket_write(mp_obj_t self_in, const void *buf, mp_uint_t
     return MP_STREAM_ERROR;
 }
 
+STATIC err_t _lwip_tcp_close_poll(void *arg, struct tcp_pcb *pcb) {
+    // Connection has not been cleanly closed so just abort it to free up memory
+    tcp_poll(pcb, NULL, 0);
+    tcp_abort(pcb);
+    return ERR_OK;
+}
+
 STATIC mp_uint_t lwip_socket_ioctl(mp_obj_t self_in, mp_uint_t request, uintptr_t arg, int *errcode) {
     lwip_socket_obj_t *socket = MP_OBJ_TO_PTR(self_in);
     mp_uint_t ret;
 
+    MICROPY_PY_LWIP_ENTER
+
     if (request == MP_STREAM_POLL) {
         uintptr_t flags = arg;
         ret = 0;
@@ -1218,10 +1366,8 @@ STATIC mp_uint_t lwip_socket_ioctl(mp_obj_t self_in, mp_uint_t request, uintptr_
         if (flags & MP_STREAM_POLL_RD) {
             if (socket->state == STATE_LISTENING) {
                 // Listening TCP socket may have one or multiple connections waiting
-                if ((socket->incoming.connection.alloc == 0
-                    && socket->incoming.connection.tcp.item != NULL)
-                    || socket->incoming.connection.tcp.array[socket->incoming.connection.iget] != NULL) {
-                        ret |= MP_STREAM_POLL_RD;
+                if (lwip_socket_incoming_array(socket)[socket->incoming.connection.iget] != NULL) {
+                    ret |= MP_STREAM_POLL_RD;
                 }
             } else {
                 // Otherwise there is just one slot for incoming data
@@ -1231,9 +1377,15 @@ STATIC mp_uint_t lwip_socket_ioctl(mp_obj_t self_in, mp_uint_t request, uintptr_
             }
         }
 
-        // Note: pcb.tcp==NULL if state<0, and in this case we can't call tcp_sndbuf
-        if (flags & MP_STREAM_POLL_WR && socket->pcb.tcp != NULL && tcp_sndbuf(socket->pcb.tcp) > 0) {
-            ret |= MP_STREAM_POLL_WR;
+        if (flags & MP_STREAM_POLL_WR) {
+            if (socket->type == MOD_NETWORK_SOCK_DGRAM && socket->pcb.udp != NULL) {
+                // UDP socket is writable
+                ret |= MP_STREAM_POLL_WR;
+            } else if (socket->pcb.tcp != NULL && tcp_sndbuf(socket->pcb.tcp) > 0) {
+                // TCP socket is writable
+                // Note: pcb.tcp==NULL if state<0, and in this case we can't call tcp_sndbuf
+                ret |= MP_STREAM_POLL_WR;
+            }
         }
 
         if (socket->state == STATE_NEW) {
@@ -1254,19 +1406,26 @@ STATIC mp_uint_t lwip_socket_ioctl(mp_obj_t self_in, mp_uint_t request, uintptr_
         }
 
     } else if (request == MP_STREAM_CLOSE) {
-        bool socket_is_listener = false;
-
         if (socket->pcb.tcp == NULL) {
+            MICROPY_PY_LWIP_EXIT
             return 0;
         }
 
         // Deregister callback (pcb.tcp is set to NULL below so must deregister now)
+        tcp_arg(socket->pcb.tcp, NULL);
+        tcp_err(socket->pcb.tcp, NULL);
         tcp_recv(socket->pcb.tcp, NULL);
 
+        // Free any incoming buffers or connections that are stored
+        lwip_socket_free_incoming(socket);
+
         switch (socket->type) {
             case MOD_NETWORK_SOCK_STREAM: {
-                if (socket->pcb.tcp->state == LISTEN) {
-                    socket_is_listener = true;
+                if (socket->pcb.tcp->state != LISTEN) {
+                    // Schedule a callback to abort the connection if it's not cleanly closed after
+                    // the given timeout.  The callback must be set before calling tcp_close since
+                    // the latter may free the pcb; if it doesn't then the callback will be active.
+                    tcp_poll(socket->pcb.tcp, _lwip_tcp_close_poll, MICROPY_PY_LWIP_TCP_CLOSE_TIMEOUT_MS / 500);
                 }
                 if (tcp_close(socket->pcb.tcp) != ERR_OK) {
                     DEBUG_printf("lwip_close: had to call tcp_abort()\n");
@@ -1277,30 +1436,9 @@ STATIC mp_uint_t lwip_socket_ioctl(mp_obj_t self_in, mp_uint_t request, uintptr_
             case MOD_NETWORK_SOCK_DGRAM: udp_remove(socket->pcb.udp); break;
             //case MOD_NETWORK_SOCK_RAW: raw_remove(socket->pcb.raw); break;
         }
+
         socket->pcb.tcp = NULL;
         socket->state = _ERR_BADF;
-        if (!socket_is_listener) {
-            if (socket->incoming.pbuf != NULL) {
-                pbuf_free(socket->incoming.pbuf);
-                socket->incoming.pbuf = NULL;
-            }
-        } else {
-            uint8_t alloc = socket->incoming.connection.alloc;
-            struct tcp_pcb *volatile *tcp_array;
-            if (alloc == 0) {
-                tcp_array = &socket->incoming.connection.tcp.item;
-            } else {
-                tcp_array = socket->incoming.connection.tcp.array;
-            }
-            for (uint8_t i = 0; i < alloc; ++i) {
-                // Deregister callback and abort
-                if (tcp_array[i] != NULL) {
-                    tcp_poll(tcp_array[i], NULL, 0);
-                    tcp_abort(tcp_array[i]);
-                    tcp_array[i] = NULL;
-                }
-            }
-        }
         ret = 0;
 
     } else {
@@ -1308,6 +1446,8 @@ STATIC mp_uint_t lwip_socket_ioctl(mp_obj_t self_in, mp_uint_t request, uintptr_
         ret = MP_STREAM_ERROR;
     }
 
+    MICROPY_PY_LWIP_EXIT
+
     return ret;
 }
 
@@ -1448,14 +1588,17 @@ STATIC mp_obj_t lwip_getaddrinfo(size_t n_args, const mp_obj_t *args) {
             && (type == 0 || type == MOD_NETWORK_SOCK_STREAM)
             && proto == 0
             && flags == 0)) {
-            mp_warning("unsupported getaddrinfo constraints");
+            mp_warning(MP_WARN_CAT(RuntimeWarning), "unsupported getaddrinfo constraints");
         }
     }
 
     getaddrinfo_state_t state;
     state.status = 0;
 
+    MICROPY_PY_LWIP_ENTER
     err_t ret = dns_gethostbyname(host, (ip_addr_t*)&state.ipaddr, lwip_getaddrinfo_cb, &state);
+    MICROPY_PY_LWIP_EXIT
+
     switch (ret) {
         case ERR_OK:
             // cached
@@ -1494,7 +1637,7 @@ STATIC mp_obj_t lwip_print_pcbs() {
 }
 MP_DEFINE_CONST_FUN_OBJ_0(lwip_print_pcbs_obj, lwip_print_pcbs);
 
-#ifdef MICROPY_PY_LWIP
+#if MICROPY_PY_LWIP
 
 STATIC const mp_rom_map_elem_t mp_module_lwip_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_lwip) },

extmod/moducryptolib.c

@@ -41,10 +41,17 @@
 
 // values follow PEP 272
 enum {
-    UCRYPTOLIB_MODE_MIN = 0,
-    UCRYPTOLIB_MODE_ECB,
-    UCRYPTOLIB_MODE_CBC,
-    UCRYPTOLIB_MODE_MAX,
+    UCRYPTOLIB_MODE_ECB = 1,
+    UCRYPTOLIB_MODE_CBC = 2,
+    UCRYPTOLIB_MODE_CTR = 6,
+};
+
+struct ctr_params {
+    // counter is the IV of the AES context.
+
+    size_t offset; // in encrypted_counter
+    // encrypted counter
+    uint8_t encrypted_counter[16];
 };
 
 #if MICROPY_SSL_AXTLS
@@ -82,6 +89,19 @@ typedef struct _mp_obj_aes_t {
     uint8_t key_type: 2;
 } mp_obj_aes_t;
 
+STATIC inline bool is_ctr_mode(int block_mode) {
+    #if MICROPY_PY_UCRYPTOLIB_CTR
+    return block_mode == UCRYPTOLIB_MODE_CTR;
+    #else
+    return false;
+    #endif
+}
+
+STATIC inline struct ctr_params *ctr_params_from_aes(mp_obj_aes_t *o) {
+    // ctr_params follows aes object struct
+    return (struct ctr_params*)&o[1];
+}
+
 #if MICROPY_SSL_AXTLS
 STATIC void aes_initial_set_key_impl(AES_CTX_IMPL *ctx, const uint8_t *key, size_t keysize, const uint8_t iv[16]) {
     assert(16 == keysize || 32 == keysize);
@@ -119,6 +139,33 @@ STATIC void aes_process_cbc_impl(AES_CTX_IMPL *ctx, const uint8_t *in, uint8_t *
         AES_cbc_decrypt(ctx, in, out, in_len);
     }
 }
+
+#if MICROPY_PY_UCRYPTOLIB_CTR
+// axTLS doesn't have CTR support out of the box. This implements the counter part using the ECB primitive.
+STATIC void aes_process_ctr_impl(AES_CTX_IMPL *ctx, const uint8_t *in, uint8_t *out, size_t in_len, struct ctr_params *ctr_params) {
+    size_t n = ctr_params->offset;
+    uint8_t *const counter = ctx->iv;
+
+    while (in_len--) {
+        if (n == 0) {
+            aes_process_ecb_impl(ctx, counter, ctr_params->encrypted_counter, true);
+
+            // increment the 128-bit counter
+            for (int i = 15; i >= 0; --i) {
+                if (++counter[i] != 0) {
+                    break;
+                }
+            }
+        }
+
+        *out++ = *in++ ^ ctr_params->encrypted_counter[n];
+        n = (n + 1) & 0xf;
+    }
+
+    ctr_params->offset = n;
+}
+#endif
+
 #endif
 
 #if MICROPY_SSL_MBEDTLS
@@ -155,20 +202,38 @@ STATIC void aes_process_ecb_impl(AES_CTX_IMPL *ctx, const uint8_t in[16], uint8_
 STATIC void aes_process_cbc_impl(AES_CTX_IMPL *ctx, const uint8_t *in, uint8_t *out, size_t in_len, bool encrypt) {
     mbedtls_aes_crypt_cbc(&ctx->u.mbedtls_ctx, encrypt ? MBEDTLS_AES_ENCRYPT : MBEDTLS_AES_DECRYPT, in_len, ctx->iv, in, out);
 }
+
+#if MICROPY_PY_UCRYPTOLIB_CTR
+STATIC void aes_process_ctr_impl(AES_CTX_IMPL *ctx, const uint8_t *in, uint8_t *out, size_t in_len, struct ctr_params *ctr_params) {
+    mbedtls_aes_crypt_ctr(&ctx->u.mbedtls_ctx, in_len, &ctr_params->offset, ctx->iv, ctr_params->encrypted_counter, in, out);
+}
+#endif
+
 #endif
 
 STATIC mp_obj_t ucryptolib_aes_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) {
     mp_arg_check_num(n_args, n_kw, 2, 3, false);
-    mp_obj_aes_t *o = m_new_obj(mp_obj_aes_t);
+
+    const mp_int_t block_mode = mp_obj_get_int(args[1]);
+
+    switch (block_mode) {
+        case UCRYPTOLIB_MODE_ECB:
+        case UCRYPTOLIB_MODE_CBC:
+        #if MICROPY_PY_UCRYPTOLIB_CTR
+        case UCRYPTOLIB_MODE_CTR:
+        #endif
+            break;
+
+        default:
+            mp_raise_ValueError("mode");
+    }
+
+    mp_obj_aes_t *o = m_new_obj_var(mp_obj_aes_t, struct ctr_params, !!is_ctr_mode(block_mode));
     o->base.type = type;
 
-    o->block_mode = mp_obj_get_int(args[1]);
+    o->block_mode = block_mode;
     o->key_type = AES_KEYTYPE_NONE;
 
-    if (o->block_mode <= UCRYPTOLIB_MODE_MIN || o->block_mode >= UCRYPTOLIB_MODE_MAX) {
-        mp_raise_ValueError("mode");
-    }
-
     mp_buffer_info_t keyinfo;
     mp_get_buffer_raise(args[0], &keyinfo, MP_BUFFER_READ);
     if (32 != keyinfo.len && 16 != keyinfo.len) {
@@ -183,10 +248,14 @@ STATIC mp_obj_t ucryptolib_aes_make_new(const mp_obj_type_t *type, size_t n_args
         if (16 != ivinfo.len) {
             mp_raise_ValueError("IV");
         }
-    } else if (o->block_mode == UCRYPTOLIB_MODE_CBC) {
+    } else if (o->block_mode == UCRYPTOLIB_MODE_CBC || is_ctr_mode(o->block_mode)) {
         mp_raise_ValueError("IV");
     }
 
+    if (is_ctr_mode(block_mode)) {
+        ctr_params_from_aes(o)->offset = 0;
+    }
+
     aes_initial_set_key_impl(&o->ctx, keyinfo.buf, keyinfo.len, ivinfo.buf);
 
     return MP_OBJ_FROM_PTR(o);
@@ -204,7 +273,7 @@ STATIC mp_obj_t aes_process(size_t n_args, const mp_obj_t *args, bool encrypt) {
     mp_buffer_info_t in_bufinfo;
     mp_get_buffer_raise(in_buf, &in_bufinfo, MP_BUFFER_READ);
 
-    if (in_bufinfo.len % 16 != 0) {
+    if (!is_ctr_mode(self->block_mode) && in_bufinfo.len % 16 != 0) {
         mp_raise_ValueError("blksize % 16");
     }
 
@@ -224,7 +293,9 @@ STATIC mp_obj_t aes_process(size_t n_args, const mp_obj_t *args, bool encrypt) {
     }
 
     if (AES_KEYTYPE_NONE == self->key_type) {
-        aes_final_set_key_impl(&self->ctx, encrypt);
+        // always set key for encryption if CTR mode.
+        const bool encrypt_mode = encrypt || is_ctr_mode(self->block_mode);
+        aes_final_set_key_impl(&self->ctx, encrypt_mode);
         self->key_type = encrypt ? AES_KEYTYPE_ENC : AES_KEYTYPE_DEC;
     } else {
         if ((encrypt && self->key_type == AES_KEYTYPE_DEC) ||
@@ -234,14 +305,26 @@ STATIC mp_obj_t aes_process(size_t n_args, const mp_obj_t *args, bool encrypt) {
         }
     }
 
-    if (self->block_mode == UCRYPTOLIB_MODE_ECB) {
-        uint8_t *in = in_bufinfo.buf, *out = out_buf_ptr;
-        uint8_t *top = in + in_bufinfo.len;
-        for (; in < top; in += 16, out += 16) {
-            aes_process_ecb_impl(&self->ctx, in, out, encrypt);
+    switch (self->block_mode) {
+        case UCRYPTOLIB_MODE_ECB: {
+            uint8_t *in = in_bufinfo.buf, *out = out_buf_ptr;
+            uint8_t *top = in + in_bufinfo.len;
+            for (; in < top; in += 16, out += 16) {
+                aes_process_ecb_impl(&self->ctx, in, out, encrypt);
+            }
+            break;
         }
-    } else {
-        aes_process_cbc_impl(&self->ctx, in_bufinfo.buf, out_buf_ptr, in_bufinfo.len, encrypt);
+
+        case UCRYPTOLIB_MODE_CBC:
+            aes_process_cbc_impl(&self->ctx, in_bufinfo.buf, out_buf_ptr, in_bufinfo.len, encrypt);
+            break;
+
+        #if MICROPY_PY_UCRYPTOLIB_CTR
+        case UCRYPTOLIB_MODE_CTR:
+            aes_process_ctr_impl(&self->ctx, in_bufinfo.buf, out_buf_ptr, in_bufinfo.len,
+                ctr_params_from_aes(self));
+            break;
+        #endif
     }
 
     if (out_buf != MP_OBJ_NULL) {
@@ -279,6 +362,9 @@ STATIC const mp_rom_map_elem_t mp_module_ucryptolib_globals_table[] = {
 #if MICROPY_PY_UCRYPTOLIB_CONSTS
     { MP_ROM_QSTR(MP_QSTR_MODE_ECB), MP_ROM_INT(UCRYPTOLIB_MODE_ECB) },
     { MP_ROM_QSTR(MP_QSTR_MODE_CBC), MP_ROM_INT(UCRYPTOLIB_MODE_CBC) },
+    #if MICROPY_PY_UCRYPTOLIB_CTR
+    { MP_ROM_QSTR(MP_QSTR_MODE_CTR), MP_ROM_INT(UCRYPTOLIB_MODE_CTR) },
+    #endif
 #endif
 };
 

extmod/moductypes.c

@@ -137,13 +137,13 @@ STATIC void uctypes_struct_print(const mp_print_t *print, mp_obj_t self_in, mp_p
     (void)kind;
     mp_obj_uctypes_struct_t *self = MP_OBJ_TO_PTR(self_in);
     const char *typen = "unk";
-    if (MP_OBJ_IS_TYPE(self->desc, &mp_type_dict)
+    if (mp_obj_is_type(self->desc, &mp_type_dict)
       #if MICROPY_PY_COLLECTIONS_ORDEREDDICT
-        || MP_OBJ_IS_TYPE(self->desc, &mp_type_ordereddict)
+        || mp_obj_is_type(self->desc, &mp_type_ordereddict)
       #endif
       ) {
         typen = "STRUCT";
-    } else if (MP_OBJ_IS_TYPE(self->desc, &mp_type_tuple)) {
+    } else if (mp_obj_is_type(self->desc, &mp_type_tuple)) {
         mp_obj_tuple_t *t = MP_OBJ_TO_PTR(self->desc);
         mp_int_t offset = MP_OBJ_SMALL_INT_VALUE(t->items[0]);
         uint agg_type = GET_TYPE(offset, AGG_TYPE_BITS);
@@ -210,14 +210,14 @@ STATIC mp_uint_t uctypes_struct_agg_size(mp_obj_tuple_t *t, int layout_type, mp_
 }
 
 STATIC mp_uint_t uctypes_struct_size(mp_obj_t desc_in, int layout_type, mp_uint_t *max_field_size) {
-    if (!MP_OBJ_IS_TYPE(desc_in, &mp_type_dict)
+    if (!mp_obj_is_type(desc_in, &mp_type_dict)
       #if MICROPY_PY_COLLECTIONS_ORDEREDDICT
-        && !MP_OBJ_IS_TYPE(desc_in, &mp_type_ordereddict)
+        && !mp_obj_is_type(desc_in, &mp_type_ordereddict)
       #endif
       ) {
-        if (MP_OBJ_IS_TYPE(desc_in, &mp_type_tuple)) {
+        if (mp_obj_is_type(desc_in, &mp_type_tuple)) {
             return uctypes_struct_agg_size((mp_obj_tuple_t*)MP_OBJ_TO_PTR(desc_in), layout_type, max_field_size);
-        } else if (MP_OBJ_IS_SMALL_INT(desc_in)) {
+        } else if (mp_obj_is_small_int(desc_in)) {
             // We allow sizeof on both type definitions and structures/structure fields,
             // but scalar structure field is lowered into native Python int, so all
             // type info is lost. So, we cannot say if it's scalar type description,
@@ -231,9 +231,9 @@ STATIC mp_uint_t uctypes_struct_size(mp_obj_t desc_in, int layout_type, mp_uint_
     mp_uint_t total_size = 0;
 
     for (mp_uint_t i = 0; i < d->map.alloc; i++) {
-        if (MP_MAP_SLOT_IS_FILLED(&d->map, i)) {
+        if (mp_map_slot_is_filled(&d->map, i)) {
             mp_obj_t v = d->map.table[i].value;
-            if (MP_OBJ_IS_SMALL_INT(v)) {
+            if (mp_obj_is_small_int(v)) {
                 mp_uint_t offset = MP_OBJ_SMALL_INT_VALUE(v);
                 mp_uint_t val_type = GET_TYPE(offset, VAL_TYPE_BITS);
                 offset &= VALUE_MASK(VAL_TYPE_BITS);
@@ -248,7 +248,7 @@ STATIC mp_uint_t uctypes_struct_size(mp_obj_t desc_in, int layout_type, mp_uint_
                     total_size = offset + s;
                 }
             } else {
-                if (!MP_OBJ_IS_TYPE(v, &mp_type_tuple)) {
+                if (!mp_obj_is_type(v, &mp_type_tuple)) {
                     syntax_error();
                 }
                 mp_obj_tuple_t *t = MP_OBJ_TO_PTR(v);
@@ -272,13 +272,13 @@ STATIC mp_uint_t uctypes_struct_size(mp_obj_t desc_in, int layout_type, mp_uint_
 STATIC mp_obj_t uctypes_struct_sizeof(size_t n_args, const mp_obj_t *args) {
     mp_obj_t obj_in = args[0];
     mp_uint_t max_field_size = 0;
-    if (MP_OBJ_IS_TYPE(obj_in, &mp_type_bytearray)) {
+    if (mp_obj_is_type(obj_in, &mp_type_bytearray)) {
         return mp_obj_len(obj_in);
     }
     int layout_type = LAYOUT_NATIVE;
     // We can apply sizeof either to structure definition (a dict)
     // or to instantiated structure
-    if (MP_OBJ_IS_TYPE(obj_in, &uctypes_struct_type)) {
+    if (mp_obj_is_type(obj_in, &uctypes_struct_type)) {
         if (n_args != 1) {
             mp_raise_TypeError(NULL);
         }
@@ -406,16 +406,16 @@ STATIC void set_aligned(uint val_type, void *p, mp_int_t index, mp_obj_t val) {
 STATIC mp_obj_t uctypes_struct_attr_op(mp_obj_t self_in, qstr attr, mp_obj_t set_val) {
     mp_obj_uctypes_struct_t *self = MP_OBJ_TO_PTR(self_in);
 
-    if (!MP_OBJ_IS_TYPE(self->desc, &mp_type_dict)
+    if (!mp_obj_is_type(self->desc, &mp_type_dict)
       #if MICROPY_PY_COLLECTIONS_ORDEREDDICT
-        && !MP_OBJ_IS_TYPE(self->desc, &mp_type_ordereddict)
+        && !mp_obj_is_type(self->desc, &mp_type_ordereddict)
       #endif
       ) {
             mp_raise_TypeError("struct: no fields");
     }
 
     mp_obj_t deref = mp_obj_dict_get(self->desc, MP_OBJ_NEW_QSTR(attr));
-    if (MP_OBJ_IS_SMALL_INT(deref)) {
+    if (mp_obj_is_small_int(deref)) {
         mp_int_t offset = MP_OBJ_SMALL_INT_VALUE(deref);
         mp_uint_t val_type = GET_TYPE(offset, VAL_TYPE_BITS);
         offset &= VALUE_MASK(VAL_TYPE_BITS);
@@ -476,7 +476,7 @@ STATIC mp_obj_t uctypes_struct_attr_op(mp_obj_t self_in, qstr attr, mp_obj_t set
         return MP_OBJ_NULL;
     }
 
-    if (!MP_OBJ_IS_TYPE(deref, &mp_type_tuple)) {
+    if (!mp_obj_is_type(deref, &mp_type_tuple)) {
         syntax_error();
     }
 
@@ -543,7 +543,7 @@ STATIC mp_obj_t uctypes_struct_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_ob
         return MP_OBJ_NULL; // op not supported
     } else {
         // load / store
-        if (!MP_OBJ_IS_TYPE(self->desc, &mp_type_tuple)) {
+        if (!mp_obj_is_type(self->desc, &mp_type_tuple)) {
             mp_raise_TypeError("struct: cannot index");
         }
 
@@ -594,7 +594,7 @@ STATIC mp_obj_t uctypes_struct_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_ob
 
         } else if (agg_type == PTR) {
             byte *p = *(void**)self->addr;
-            if (MP_OBJ_IS_SMALL_INT(t->items[1])) {
+            if (mp_obj_is_small_int(t->items[1])) {
                 uint val_type = GET_TYPE(MP_OBJ_SMALL_INT_VALUE(t->items[1]), VAL_TYPE_BITS);
                 return get_aligned(val_type, p, index);
             } else {
@@ -618,7 +618,7 @@ STATIC mp_obj_t uctypes_struct_unary_op(mp_unary_op_t op, mp_obj_t self_in) {
     mp_obj_uctypes_struct_t *self = MP_OBJ_TO_PTR(self_in);
     switch (op) {
         case MP_UNARY_OP_INT:
-            if (MP_OBJ_IS_TYPE(self->desc, &mp_type_tuple)) {
+            if (mp_obj_is_type(self->desc, &mp_type_tuple)) {
                 mp_obj_tuple_t *t = MP_OBJ_TO_PTR(self->desc);
                 mp_int_t offset = MP_OBJ_SMALL_INT_VALUE(t->items[0]);
                 uint agg_type = GET_TYPE(offset, AGG_TYPE_BITS);

extmod/moduhashlib.c

@@ -104,6 +104,8 @@ STATIC mp_obj_t uhashlib_sha256_digest(mp_obj_t self_in) {
 
 #else
 
+#include "crypto-algorithms/sha256.c"
+
 STATIC mp_obj_t uhashlib_sha256_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) {
     mp_arg_check_num(n_args, n_kw, 0, 1, false);
     mp_obj_hash_t *o = m_new_obj_var(mp_obj_hash_t, char, sizeof(CRYAL_SHA256_CTX));
@@ -344,8 +346,4 @@ const mp_obj_module_t mp_module_uhashlib = {
     .globals = (mp_obj_dict_t*)&mp_module_uhashlib_globals,
 };
 
-#if MICROPY_PY_UHASHLIB_SHA256
-#include "crypto-algorithms/sha256.c"
-#endif
-
 #endif //MICROPY_PY_UHASHLIB

extmod/moduheapq.c

@@ -32,7 +32,7 @@
 // the algorithm here is modelled on CPython's heapq.py
 
 STATIC mp_obj_list_t *get_heap(mp_obj_t heap_in) {
-    if (!MP_OBJ_IS_TYPE(heap_in, &mp_type_list)) {
+    if (!mp_obj_is_type(heap_in, &mp_type_list)) {
         mp_raise_TypeError("heap must be a list");
     }
     return MP_OBJ_TO_PTR(heap_in);

extmod/modujson.c

@@ -185,7 +185,7 @@ STATIC mp_obj_t mod_ujson_load(mp_obj_t stream_obj) {
                     cur = S_CUR(s);
                     if (cur == '.' || cur == 'E' || cur == 'e') {
                         flt = true;
-                    } else if (cur == '-' || unichar_isdigit(cur)) {
+                    } else if (cur == '+' || cur == '-' || unichar_isdigit(cur)) {
                         // pass
                     } else {
                         break;

extmod/modurandom.c

@@ -200,8 +200,19 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_2(mod_urandom_uniform_obj, mod_urandom_uniform);
 
 #endif // MICROPY_PY_URANDOM_EXTRA_FUNCS
 
+#ifdef MICROPY_PY_URANDOM_SEED_INIT_FUNC
+STATIC mp_obj_t mod_urandom___init__() {
+    mod_urandom_seed(MP_OBJ_NEW_SMALL_INT(MICROPY_PY_URANDOM_SEED_INIT_FUNC));
+    return mp_const_none;
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_0(mod_urandom___init___obj, mod_urandom___init__);
+#endif
+
 STATIC const mp_rom_map_elem_t mp_module_urandom_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_urandom) },
+    #ifdef MICROPY_PY_URANDOM_SEED_INIT_FUNC
+    { MP_ROM_QSTR(MP_QSTR___init__), MP_ROM_PTR(&mod_urandom___init___obj) },
+    #endif
     { MP_ROM_QSTR(MP_QSTR_getrandbits), MP_ROM_PTR(&mod_urandom_getrandbits_obj) },
     { MP_ROM_QSTR(MP_QSTR_seed), MP_ROM_PTR(&mod_urandom_seed_obj) },
     #if MICROPY_PY_URANDOM_EXTRA_FUNCS

extmod/moduselect.c

@@ -4,6 +4,7 @@
  * The MIT License (MIT)
  *
  * Copyright (c) 2014 Damien P. George
+ * Copyright (c) 2015-2017 Paul Sokolovsky
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -77,7 +78,7 @@ STATIC void poll_map_add(mp_map_t *poll_map, const mp_obj_t *obj, mp_uint_t obj_
 STATIC mp_uint_t poll_map_poll(mp_map_t *poll_map, size_t *rwx_num) {
     mp_uint_t n_ready = 0;
     for (mp_uint_t i = 0; i < poll_map->alloc; ++i) {
-        if (!MP_MAP_SLOT_IS_FILLED(poll_map, i)) {
+        if (!mp_map_slot_is_filled(poll_map, i)) {
             continue;
         }
 
@@ -111,7 +112,7 @@ STATIC mp_uint_t poll_map_poll(mp_map_t *poll_map, size_t *rwx_num) {
 }
 
 /// \function select(rlist, wlist, xlist[, timeout])
-STATIC mp_obj_t select_select(uint n_args, const mp_obj_t *args) {
+STATIC mp_obj_t select_select(size_t n_args, const mp_obj_t *args) {
     // get array data from tuple/list arguments
     size_t rwx_len[3];
     mp_obj_t *r_array, *w_array, *x_array;
@@ -155,7 +156,7 @@ STATIC mp_obj_t select_select(uint n_args, const mp_obj_t *args) {
             list_array[2] = mp_obj_new_list(rwx_len[2], NULL);
             rwx_len[0] = rwx_len[1] = rwx_len[2] = 0;
             for (mp_uint_t i = 0; i < poll_map.alloc; ++i) {
-                if (!MP_MAP_SLOT_IS_FILLED(&poll_map, i)) {
+                if (!mp_map_slot_is_filled(&poll_map, i)) {
                     continue;
                 }
                 poll_obj_t *poll_obj = MP_OBJ_TO_PTR(poll_map.table[i].value);
@@ -190,7 +191,7 @@ typedef struct _mp_obj_poll_t {
 } mp_obj_poll_t;
 
 /// \method register(obj[, eventmask])
-STATIC mp_obj_t poll_register(uint n_args, const mp_obj_t *args) {
+STATIC mp_obj_t poll_register(size_t n_args, const mp_obj_t *args) {
     mp_obj_poll_t *self = MP_OBJ_TO_PTR(args[0]);
     mp_uint_t flags;
     if (n_args == 3) {
@@ -266,7 +267,7 @@ STATIC mp_obj_t poll_poll(size_t n_args, const mp_obj_t *args) {
     mp_obj_list_t *ret_list = MP_OBJ_TO_PTR(mp_obj_new_list(n_ready, NULL));
     n_ready = 0;
     for (mp_uint_t i = 0; i < self->poll_map.alloc; ++i) {
-        if (!MP_MAP_SLOT_IS_FILLED(&self->poll_map, i)) {
+        if (!mp_map_slot_is_filled(&self->poll_map, i)) {
             continue;
         }
         poll_obj_t *poll_obj = MP_OBJ_TO_PTR(self->poll_map.table[i].value);
@@ -309,7 +310,7 @@ STATIC mp_obj_t poll_iternext(mp_obj_t self_in) {
 
     for (mp_uint_t i = self->iter_idx; i < self->poll_map.alloc; ++i) {
         self->iter_idx++;
-        if (!MP_MAP_SLOT_IS_FILLED(&self->poll_map, i)) {
+        if (!mp_map_slot_is_filled(&self->poll_map, i)) {
             continue;
         }
         poll_obj_t *poll_obj = MP_OBJ_TO_PTR(self->poll_map.table[i].value);

extmod/modussl_axtls.c

@@ -3,7 +3,7 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2015-2017 Paul Sokolovsky
+ * Copyright (c) 2015-2019 Paul Sokolovsky
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -41,6 +41,7 @@ typedef struct _mp_obj_ssl_socket_t {
     SSL *ssl_sock;
     byte *buf;
     uint32_t bytes_left;
+    bool blocking;
 } mp_obj_ssl_socket_t;
 
 struct ssl_args {
@@ -48,6 +49,7 @@ struct ssl_args {
     mp_arg_val_t cert;
     mp_arg_val_t server_side;
     mp_arg_val_t server_hostname;
+    mp_arg_val_t do_handshake;
 };
 
 STATIC const mp_obj_type_t ussl_socket_type;
@@ -62,8 +64,12 @@ STATIC mp_obj_ssl_socket_t *socket_new(mp_obj_t sock, struct ssl_args *args) {
     o->buf = NULL;
     o->bytes_left = 0;
     o->sock = sock;
+    o->blocking = true;
 
     uint32_t options = SSL_SERVER_VERIFY_LATER;
+    if (!args->do_handshake.u_bool) {
+        options |= SSL_CONNECT_IN_PARTS;
+    }
     if (args->key.u_obj != mp_const_none) {
         options |= SSL_NO_DEFAULT_KEY;
     }
@@ -97,17 +103,14 @@ STATIC mp_obj_ssl_socket_t *socket_new(mp_obj_t sock, struct ssl_args *args) {
 
         o->ssl_sock = ssl_client_new(o->ssl_ctx, (long)sock, NULL, 0, ext);
 
-        int res = ssl_handshake_status(o->ssl_sock);
-        // Pointer to SSL_EXTENSIONS as being passed to ssl_client_new()
-        // is saved in ssl_sock->extensions.
-        // As of axTLS 2.1.3, extensions aren't used beyond the initial
-        // handshake, and that's pretty much how it's expected to be. So
-        // we allocate them on stack and reset the pointer after handshake.
+        if (args->do_handshake.u_bool) {
+            int res = ssl_handshake_status(o->ssl_sock);
 
-        if (res != SSL_OK) {
-            printf("ssl_handshake_status: %d\n", res);
-            ssl_display_error(res);
-            mp_raise_OSError(MP_EIO);
+            if (res != SSL_OK) {
+                printf("ssl_handshake_status: %d\n", res);
+                ssl_display_error(res);
+                mp_raise_OSError(MP_EIO);
+            }
         }
 
     }
@@ -133,8 +136,18 @@ STATIC mp_uint_t socket_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errc
         mp_int_t r = ssl_read(o->ssl_sock, &o->buf);
         if (r == SSL_OK) {
             // SSL_OK from ssl_read() means "everything is ok, but there's
-            // no user data yet". So, we just keep reading.
-            continue;
+            // no user data yet". It may happen e.g. if handshake is not
+            // finished yet. The best way we can treat it is by returning
+            // EAGAIN. This may be a bit unexpected in blocking mode, but
+            // default is to perform complete handshake in constructor, so
+            // this should not happen in blocking mode. On the other hand,
+            // in nonblocking mode EAGAIN (comparing to the alternative of
+            // looping) is really preferrable.
+            if (o->blocking) {
+                continue;
+            } else {
+                goto eagain;
+            }
         }
         if (r < 0) {
             if (r == SSL_CLOSE_NOTIFY || r == SSL_ERROR_CONN_LOST) {
@@ -142,6 +155,7 @@ STATIC mp_uint_t socket_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errc
                 return 0;
             }
             if (r == SSL_EAGAIN) {
+eagain:
                 r = MP_EAGAIN;
             }
             *errcode = r;
@@ -187,12 +201,14 @@ STATIC mp_uint_t socket_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, i
 }
 
 STATIC mp_obj_t socket_setblocking(mp_obj_t self_in, mp_obj_t flag_in) {
-    // Currently supports only blocking mode
-    (void)self_in;
-    if (!mp_obj_is_true(flag_in)) {
-        mp_raise_NotImplementedError(NULL);
-    }
-    return mp_const_none;
+    mp_obj_ssl_socket_t *o = MP_OBJ_TO_PTR(self_in);
+    mp_obj_t sock = o->sock;
+    mp_obj_t dest[3];
+    mp_load_method(sock, MP_QSTR_setblocking, dest);
+    dest[2] = flag_in;
+    mp_obj_t res = mp_call_method_n_kw(1, 0, dest);
+    o->blocking = mp_obj_is_true(flag_in);
+    return res;
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_2(socket_setblocking_obj, socket_setblocking);
 
@@ -234,6 +250,7 @@ STATIC mp_obj_t mod_ssl_wrap_socket(size_t n_args, const mp_obj_t *pos_args, mp_
         { MP_QSTR_cert, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = MP_ROM_PTR(&mp_const_none_obj)} },
         { MP_QSTR_server_side, MP_ARG_KW_ONLY | MP_ARG_BOOL, {.u_bool = false} },
         { MP_QSTR_server_hostname, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = MP_ROM_PTR(&mp_const_none_obj)} },
+        { MP_QSTR_do_handshake, MP_ARG_KW_ONLY | MP_ARG_BOOL, {.u_bool = true} },
     };
 
     // TODO: Check that sock implements stream protocol

extmod/modussl_mbedtls.c

@@ -4,6 +4,7 @@
  * The MIT License (MIT)
  *
  * Copyright (c) 2016 Linaro Ltd.
+ * Copyright (c) 2019 Paul Sokolovsky
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -36,7 +37,6 @@
 
 // mbedtls_time_t
 #include "mbedtls/platform.h"
-#include "mbedtls/net.h"
 #include "mbedtls/ssl.h"
 #include "mbedtls/x509_crt.h"
 #include "mbedtls/pk.h"
@@ -61,6 +61,7 @@ struct ssl_args {
     mp_arg_val_t cert;
     mp_arg_val_t server_side;
     mp_arg_val_t server_hostname;
+    mp_arg_val_t do_handshake;
 };
 
 STATIC const mp_obj_type_t ussl_socket_type;
@@ -185,10 +186,12 @@ STATIC mp_obj_ssl_socket_t *socket_new(mp_obj_t sock, struct ssl_args *args) {
         assert(ret == 0);
     }
 
-    while ((ret = mbedtls_ssl_handshake(&o->ssl)) != 0) {
-        if (ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE) {
-            printf("mbedtls_ssl_handshake error: -%x\n", -ret);
-            goto cleanup;
+    if (args->do_handshake.u_bool) {
+        while ((ret = mbedtls_ssl_handshake(&o->ssl)) != 0) {
+            if (ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE) {
+                printf("mbedtls_ssl_handshake error: -%x\n", -ret);
+                goto cleanup;
+            }
         }
     }
 
@@ -239,6 +242,11 @@ STATIC mp_uint_t socket_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errc
     }
     if (ret == MBEDTLS_ERR_SSL_WANT_READ) {
         ret = MP_EWOULDBLOCK;
+    } else if (ret == MBEDTLS_ERR_SSL_WANT_WRITE) {
+        // If handshake is not finished, read attempt may end up in protocol
+        // wanting to write next handshake message. The same may happen with
+        // renegotation.
+        ret = MP_EWOULDBLOCK;
     }
     *errcode = ret;
     return MP_STREAM_ERROR;
@@ -253,6 +261,11 @@ STATIC mp_uint_t socket_write(mp_obj_t o_in, const void *buf, mp_uint_t size, in
     }
     if (ret == MBEDTLS_ERR_SSL_WANT_WRITE) {
         ret = MP_EWOULDBLOCK;
+    } else if (ret == MBEDTLS_ERR_SSL_WANT_READ) {
+        // If handshake is not finished, write attempt may end up in protocol
+        // wanting to read next handshake message. The same may happen with
+        // renegotation.
+        ret = MP_EWOULDBLOCK;
     }
     *errcode = ret;
     return MP_STREAM_ERROR;
@@ -322,6 +335,7 @@ STATIC mp_obj_t mod_ssl_wrap_socket(size_t n_args, const mp_obj_t *pos_args, mp_
         { MP_QSTR_cert, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL} },
         { MP_QSTR_server_side, MP_ARG_KW_ONLY | MP_ARG_BOOL, {.u_bool = false} },
         { MP_QSTR_server_hostname, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = mp_const_none} },
+        { MP_QSTR_do_handshake, MP_ARG_KW_ONLY | MP_ARG_BOOL, {.u_bool = true} },
     };
 
     // TODO: Check that sock implements stream protocol

extmod/modutimeq.c

@@ -145,7 +145,7 @@ STATIC mp_obj_t mod_utimeq_heappop(mp_obj_t heap_in, mp_obj_t list_ref) {
         nlr_raise(mp_obj_new_exception_msg(&mp_type_IndexError, "empty heap"));
     }
     mp_obj_list_t *ret = MP_OBJ_TO_PTR(list_ref);
-    if (!MP_OBJ_IS_TYPE(list_ref, &mp_type_list) || ret->len < 3) {
+    if (!mp_obj_is_type(list_ref, &mp_type_list) || ret->len < 3) {
         mp_raise_TypeError(NULL);
     }
 

extmod/moduwebsocket.c

@@ -30,9 +30,9 @@
 
 #include "py/runtime.h"
 #include "py/stream.h"
-#include "extmod/modwebsocket.h"
+#include "extmod/moduwebsocket.h"
 
-#if MICROPY_PY_WEBSOCKET
+#if MICROPY_PY_UWEBSOCKET
 
 enum { FRAME_HEADER, FRAME_OPT, PAYLOAD, CONTROL };
 
@@ -299,16 +299,16 @@ STATIC const mp_obj_type_t websocket_type = {
     .locals_dict = (void*)&websocket_locals_dict,
 };
 
-STATIC const mp_rom_map_elem_t websocket_module_globals_table[] = {
-    { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_websocket) },
+STATIC const mp_rom_map_elem_t uwebsocket_module_globals_table[] = {
+    { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uwebsocket) },
     { MP_ROM_QSTR(MP_QSTR_websocket), MP_ROM_PTR(&websocket_type) },
 };
 
-STATIC MP_DEFINE_CONST_DICT(websocket_module_globals, websocket_module_globals_table);
+STATIC MP_DEFINE_CONST_DICT(uwebsocket_module_globals, uwebsocket_module_globals_table);
 
-const mp_obj_module_t mp_module_websocket = {
+const mp_obj_module_t mp_module_uwebsocket = {
     .base = { &mp_type_module },
-    .globals = (mp_obj_dict_t*)&websocket_module_globals,
+    .globals = (mp_obj_dict_t*)&uwebsocket_module_globals,
 };
 
-#endif // MICROPY_PY_WEBSOCKET
+#endif // MICROPY_PY_UWEBSOCKET

extmod/moduwebsocket.h

@@ -0,0 +1,10 @@
+#ifndef MICROPY_INCLUDED_EXTMOD_MODUWEBSOCKET_H
+#define MICROPY_INCLUDED_EXTMOD_MODUWEBSOCKET_H
+
+#define FRAME_OPCODE_MASK 0x0f
+enum {
+    FRAME_CONT, FRAME_TXT, FRAME_BIN,
+    FRAME_CLOSE = 0x8, FRAME_PING, FRAME_PONG
+};
+
+#endif // MICROPY_INCLUDED_EXTMOD_MODUWEBSOCKET_H

extmod/moduzlib.c

@@ -48,7 +48,7 @@ typedef struct _mp_obj_decompio_t {
     bool eof;
 } mp_obj_decompio_t;
 
-STATIC unsigned char read_src_stream(TINF_DATA *data) {
+STATIC int read_src_stream(TINF_DATA *data) {
     byte *p = (void*)data;
     p -= offsetof(mp_obj_decompio_t, decomp);
     mp_obj_decompio_t *self = (mp_obj_decompio_t*)p;
@@ -110,7 +110,7 @@ STATIC mp_uint_t decompio_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *er
     }
 
     o->decomp.dest = buf;
-    o->decomp.destSize = size;
+    o->decomp.dest_limit = (byte*)buf + size;
     int st = uzlib_uncompress_chksum(&o->decomp);
     if (st == TINF_DONE) {
         o->eof = true;
@@ -155,9 +155,10 @@ STATIC mp_obj_t mod_uzlib_decompress(size_t n_args, const mp_obj_t *args) {
     byte *dest_buf = m_new(byte, dest_buf_size);
 
     decomp->dest = dest_buf;
-    decomp->destSize = dest_buf_size;
-    DEBUG_printf("uzlib: Initial out buffer: " UINT_FMT " bytes\n", decomp->destSize);
+    decomp->dest_limit = dest_buf + dest_buf_size;
+    DEBUG_printf("uzlib: Initial out buffer: " UINT_FMT " bytes\n", dest_buf_size);
     decomp->source = bufinfo.buf;
+    decomp->source_limit = (byte*)bufinfo.buf + bufinfo.len;
 
     int st;
     bool is_zlib = true;
@@ -185,7 +186,7 @@ STATIC mp_obj_t mod_uzlib_decompress(size_t n_args, const mp_obj_t *args) {
         dest_buf = m_renew(byte, dest_buf, dest_buf_size, dest_buf_size + 256);
         dest_buf_size += 256;
         decomp->dest = dest_buf + offset;
-        decomp->destSize = 256;
+        decomp->dest_limit = decomp->dest + 256;
     }
 
     mp_uint_t final_sz = decomp->dest - dest_buf;

extmod/modwebrepl.c

@@ -34,7 +34,7 @@
 #ifdef MICROPY_PY_WEBREPL_DELAY
 #include "py/mphal.h"
 #endif
-#include "extmod/modwebsocket.h"
+#include "extmod/moduwebsocket.h"
 
 #if MICROPY_PY_WEBREPL
 
@@ -108,6 +108,15 @@ STATIC mp_obj_t webrepl_make_new(const mp_obj_type_t *type, size_t n_args, size_
     return o;
 }
 
+STATIC void check_file_op_finished(mp_obj_webrepl_t *self) {
+    if (self->data_to_recv == 0) {
+        mp_stream_close(self->cur_file);
+        self->hdr_to_recv = sizeof(struct webrepl_file);
+        DEBUG_printf("webrepl: Finished file operation %d\n", self->hdr.type);
+        write_webrepl_resp(self->sock, 0);
+    }
+}
+
 STATIC int write_file_chunk(mp_obj_webrepl_t *self) {
     const mp_stream_p_t *file_stream = mp_get_stream(self->cur_file);
     byte readbuf[2 + 256];
@@ -160,6 +169,7 @@ STATIC void handle_op(mp_obj_webrepl_t *self) {
 
     if (self->hdr.type == PUT_FILE) {
         self->data_to_recv = self->hdr.size;
+        check_file_op_finished(self);
     } else if (self->hdr.type == GET_FILE) {
         self->data_to_recv = 1;
     }
@@ -266,12 +276,7 @@ STATIC mp_uint_t _webrepl_read(mp_obj_t self_in, void *buf, mp_uint_t size, int
             }
         }
 
-        if (self->data_to_recv == 0) {
-            mp_stream_close(self->cur_file);
-            self->hdr_to_recv = sizeof(struct webrepl_file);
-            DEBUG_printf("webrepl: Finished file operation %d\n", self->hdr.type);
-            write_webrepl_resp(self->sock, 0);
-        }
+        check_file_op_finished(self);
 
         #ifdef MICROPY_PY_WEBREPL_DELAY
         // Some platforms may have broken drivers and easily gets

extmod/modwebsocket.h

@@ -1,10 +0,0 @@
-#ifndef MICROPY_INCLUDED_EXTMOD_MODWEBSOCKET_H
-#define MICROPY_INCLUDED_EXTMOD_MODWEBSOCKET_H
-
-#define FRAME_OPCODE_MASK 0x0f
-enum {
-    FRAME_CONT, FRAME_TXT, FRAME_BIN,
-    FRAME_CLOSE = 0x8, FRAME_PING, FRAME_PONG
-};
-
-#endif // MICROPY_INCLUDED_EXTMOD_MODWEBSOCKET_H

extmod/uos_dupterm.c

@@ -32,6 +32,7 @@
 #include "py/objtuple.h"
 #include "py/objarray.h"
 #include "py/stream.h"
+#include "extmod/misc.h"
 #include "lib/utils/interrupt_char.h"
 
 #if MICROPY_PY_OS_DUPTERM
@@ -58,6 +59,20 @@ int mp_uos_dupterm_rx_chr(void) {
             continue;
         }
 
+        #if MICROPY_PY_UOS_DUPTERM_BUILTIN_STREAM
+        if (mp_uos_dupterm_is_builtin_stream(MP_STATE_VM(dupterm_objs[idx]))) {
+            byte buf[1];
+            int errcode = 0;
+            const mp_stream_p_t *stream_p = mp_get_stream(MP_STATE_VM(dupterm_objs[idx]));
+            mp_uint_t out_sz = stream_p->read(MP_STATE_VM(dupterm_objs[idx]), buf, 1, &errcode);
+            if (errcode == 0 && out_sz != 0) {
+                return buf[0];
+             } else {
+                continue;
+             }
+        }
+        #endif
+
         nlr_buf_t nlr;
         if (nlr_push(&nlr) == 0) {
             byte buf[1];
@@ -98,6 +113,16 @@ void mp_uos_dupterm_tx_strn(const char *str, size_t len) {
         if (MP_STATE_VM(dupterm_objs[idx]) == MP_OBJ_NULL) {
             continue;
         }
+
+        #if MICROPY_PY_UOS_DUPTERM_BUILTIN_STREAM
+        if (mp_uos_dupterm_is_builtin_stream(MP_STATE_VM(dupterm_objs[idx]))) {
+            int errcode = 0;
+            const mp_stream_p_t *stream_p = mp_get_stream(MP_STATE_VM(dupterm_objs[idx]));
+            stream_p->write(MP_STATE_VM(dupterm_objs[idx]), str, len, &errcode);
+            continue;
+        }
+        #endif
+
         nlr_buf_t nlr;
         if (nlr_push(&nlr) == 0) {
             mp_stream_write(MP_STATE_VM(dupterm_objs[idx]), str, len, MP_STREAM_RW_WRITE);

extmod/uzlib/defl_static.h

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) uzlib authors
+ *
+ * This software is provided 'as-is', without any express
+ * or implied warranty.  In no event will the authors be
+ * held liable for any damages arising from the use of
+ * this software.
+ *
+ * Permission is granted to anyone to use this software
+ * for any purpose, including commercial applications,
+ * and to alter it and redistribute it freely, subject to
+ * the following restrictions:
+ *
+ * 1. The origin of this software must not be
+ *    misrepresented; you must not claim that you
+ *    wrote the original software. If you use this
+ *    software in a product, an acknowledgment in
+ *    the product documentation would be appreciated
+ *    but is not required.
+ *
+ * 2. Altered source versions must be plainly marked
+ *    as such, and must not be misrepresented as
+ *    being the original software.
+ *
+ * 3. This notice may not be removed or altered from
+ *    any source distribution.
+ */
+
+/* This files contains type declaration and prototypes for defl_static.c.
+   They may be altered/distinct from the originals used in PuTTY source
+   code. */
+
+struct Outbuf {
+    unsigned char *outbuf;
+    int outlen, outsize;
+    unsigned long outbits;
+    int noutbits;
+    int comp_disabled;
+};
+
+void outbits(struct Outbuf *out, unsigned long bits, int nbits);
+void zlib_start_block(struct Outbuf *ctx);
+void zlib_finish_block(struct Outbuf *ctx);
+void zlib_literal(struct Outbuf *ectx, unsigned char c);
+void zlib_match(struct Outbuf *ectx, int distance, int len);

extmod/uzlib/tinf.h

@@ -1,117 +1,3 @@
-/*
- * uzlib  -  tiny deflate/inflate library (deflate, gzip, zlib)
- *
- * Copyright (c) 2003 by Joergen Ibsen / Jibz
- * All Rights Reserved
- * http://www.ibsensoftware.com/
- *
- * Copyright (c) 2014-2016 by Paul Sokolovsky
- */
-
-#ifndef TINF_H_INCLUDED
-#define TINF_H_INCLUDED
-
-#include <stdint.h>
-
-/* calling convention */
-#ifndef TINFCC
- #ifdef __WATCOMC__
-  #define TINFCC __cdecl
- #else
-  #define TINFCC
- #endif
-#endif
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* ok status, more data produced */
-#define TINF_OK             0
-/* end of compressed stream reached */
-#define TINF_DONE           1
-#define TINF_DATA_ERROR    (-3)
-#define TINF_CHKSUM_ERROR  (-4)
-#define TINF_DICT_ERROR    (-5)
-
-/* checksum types */
-#define TINF_CHKSUM_NONE  0
-#define TINF_CHKSUM_ADLER 1
-#define TINF_CHKSUM_CRC   2
-
-/* data structures */
-
-typedef struct {
-   unsigned short table[16];  /* table of code length counts */
-   unsigned short trans[288]; /* code -> symbol translation table */
-} TINF_TREE;
-
-struct TINF_DATA;
-typedef struct TINF_DATA {
-   const unsigned char *source;
-   /* If source above is NULL, this function will be used to read
-      next byte from source stream */
-   unsigned char (*readSource)(struct TINF_DATA *data);
-
-   unsigned int tag;
-   unsigned int bitcount;
-
-    /* Buffer start */
-    unsigned char *destStart;
-    /* Buffer total size */
-    unsigned int destSize;
-    /* Current pointer in buffer */
-    unsigned char *dest;
-    /* Remaining bytes in buffer */
-    unsigned int destRemaining;
-
-    /* Accumulating checksum */
-    unsigned int checksum;
-    char checksum_type;
-
-    int btype;
-    int bfinal;
-    unsigned int curlen;
-    int lzOff;
-    unsigned char *dict_ring;
-    unsigned int dict_size;
-    unsigned int dict_idx;
-
-   TINF_TREE ltree; /* dynamic length/symbol tree */
-   TINF_TREE dtree; /* dynamic distance tree */
-} TINF_DATA;
-
-#define TINF_PUT(d, c) \
-    { \
-        *d->dest++ = c; \
-        if (d->dict_ring) { d->dict_ring[d->dict_idx++] = c; if (d->dict_idx == d->dict_size) d->dict_idx = 0; } \
-    }
-
-unsigned char TINFCC uzlib_get_byte(TINF_DATA *d);
-
-/* Decompression API */
-
-void TINFCC uzlib_init(void);
-void TINFCC uzlib_uncompress_init(TINF_DATA *d, void *dict, unsigned int dictLen);
-int  TINFCC uzlib_uncompress(TINF_DATA *d);
-int  TINFCC uzlib_uncompress_chksum(TINF_DATA *d);
-
-int TINFCC uzlib_zlib_parse_header(TINF_DATA *d);
-int TINFCC uzlib_gzip_parse_header(TINF_DATA *d);
-
-/* Compression API */
-
-void TINFCC uzlib_compress(void *data, const uint8_t *src, unsigned slen);
-
-/* Checksum API */
-
-/* prev_sum is previous value for incremental computation, 1 initially */
-uint32_t TINFCC uzlib_adler32(const void *data, unsigned int length, uint32_t prev_sum);
-/* crc is previous value for incremental computation, 0xffffffff initially */
-uint32_t TINFCC uzlib_crc32(const void *data, unsigned int length, uint32_t crc);
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif
-
-#endif /* TINF_H_INCLUDED */
+/* Compatibility header for the original tinf lib/older versions of uzlib.
+   Note: may be removed in the future, please migrate to uzlib.h. */
+#include "uzlib.h"

extmod/uzlib/tinf_compat.h

@@ -0,0 +1,9 @@
+/* This header contains compatibility defines for the original tinf API
+   and uzlib 2.x and below API. These defines are deprecated and going
+   to be removed in the future, so applications should migrate to new
+   uzlib API. */
+#define TINF_DATA struct uzlib_uncomp
+
+#define destSize dest_size
+#define destStart dest_start
+#define readSource source_read_cb

extmod/uzlib/tinfgzip.c

@@ -1,12 +1,12 @@
 /*
- * tinfgzip  -  tiny gzip decompressor
+ * uzlib  -  tiny deflate/inflate library (deflate, gzip, zlib)
  *
  * Copyright (c) 2003 by Joergen Ibsen / Jibz
  * All Rights Reserved
  *
  * http://www.ibsensoftware.com/
  *
- * Copyright (c) 2014-2016 by Paul Sokolovsky
+ * Copyright (c) 2014-2018 by Paul Sokolovsky
  *
  * This software is provided 'as-is', without any express
  * or implied warranty.  In no event will the authors be

extmod/uzlib/tinflate.c

@@ -1,11 +1,11 @@
 /*
- * tinflate  -  tiny inflate
+ * uzlib  -  tiny deflate/inflate library (deflate, gzip, zlib)
  *
  * Copyright (c) 2003 by Joergen Ibsen / Jibz
  * All Rights Reserved
  * http://www.ibsensoftware.com/
  *
- * Copyright (c) 2014-2016 by Paul Sokolovsky
+ * Copyright (c) 2014-2018 by Paul Sokolovsky
  *
  * This software is provided 'as-is', without any express
  * or implied warranty.  In no event will the authors be
@@ -35,6 +35,15 @@
 #include <assert.h>
 #include "tinf.h"
 
+#define UZLIB_DUMP_ARRAY(heading, arr, size) \
+    { \
+        printf("%s", heading); \
+        for (int i = 0; i < size; ++i) { \
+            printf(" %d", (arr)[i]); \
+        } \
+        printf("\n"); \
+    }
+
 uint32_t tinf_get_le_uint32(TINF_DATA *d);
 uint32_t tinf_get_be_uint32(TINF_DATA *d);
 
@@ -149,6 +158,13 @@ static void tinf_build_tree(TINF_TREE *t, const unsigned char *lengths, unsigned
    /* scan symbol lengths, and sum code length counts */
    for (i = 0; i < num; ++i) t->table[lengths[i]]++;
 
+   #if UZLIB_CONF_DEBUG_LOG >= 2
+   UZLIB_DUMP_ARRAY("codelen counts:", t->table, TINF_ARRAY_SIZE(t->table));
+   #endif
+
+   /* In the lengths array, 0 means unused code. So, t->table[0] now contains
+      number of unused codes. But table's purpose is to contain # of codes of
+      particular length, and there're 0 codes of length 0. */
    t->table[0] = 0;
 
    /* compute offset table for distribution sort */
@@ -158,6 +174,10 @@ static void tinf_build_tree(TINF_TREE *t, const unsigned char *lengths, unsigned
       sum += t->table[i];
    }
 
+   #if UZLIB_CONF_DEBUG_LOG >= 2
+   UZLIB_DUMP_ARRAY("codelen offsets:", offs, TINF_ARRAY_SIZE(offs));
+   #endif
+
    /* create code->symbol translation table (symbols sorted by code) */
    for (i = 0; i < num; ++i)
    {
@@ -171,10 +191,28 @@ static void tinf_build_tree(TINF_TREE *t, const unsigned char *lengths, unsigned
 
 unsigned char uzlib_get_byte(TINF_DATA *d)
 {
-    if (d->source) {
+    /* If end of source buffer is not reached, return next byte from source
+       buffer. */
+    if (d->source < d->source_limit) {
         return *d->source++;
     }
-    return d->readSource(d);
+
+    /* Otherwise if there's callback and we haven't seen EOF yet, try to
+       read next byte using it. (Note: the callback can also update ->source
+       and ->source_limit). */
+    if (d->readSource && !d->eof) {
+        int val = d->readSource(d);
+        if (val >= 0) {
+            return (unsigned char)val;
+        }
+    }
+
+    /* Otherwise, we hit EOF (either from ->readSource() or from exhaustion
+       of the buffer), and it will be "sticky", i.e. further calls to this
+       function will end up here too. */
+    d->eof = true;
+
+    return 0;
 }
 
 uint32_t tinf_get_le_uint32(TINF_DATA *d)
@@ -182,7 +220,7 @@ uint32_t tinf_get_le_uint32(TINF_DATA *d)
     uint32_t val = 0;
     int i;
     for (i = 4; i--;) {
-        val = val >> 8 | uzlib_get_byte(d) << 24;
+        val = val >> 8 | ((uint32_t)uzlib_get_byte(d)) << 24;
     }
     return val;
 }
@@ -245,21 +283,31 @@ static int tinf_decode_symbol(TINF_DATA *d, TINF_TREE *t)
 
       cur = 2*cur + tinf_getbit(d);
 
-      ++len;
+      if (++len == TINF_ARRAY_SIZE(t->table)) {
+         return TINF_DATA_ERROR;
+      }
 
       sum += t->table[len];
       cur -= t->table[len];
 
    } while (cur >= 0);
 
-   return t->trans[sum + cur];
+   sum += cur;
+   #if UZLIB_CONF_PARANOID_CHECKS
+   if (sum < 0 || sum >= TINF_ARRAY_SIZE(t->trans)) {
+      return TINF_DATA_ERROR;
+   }
+   #endif
+
+   return t->trans[sum];
 }
 
 /* given a data stream, decode dynamic trees from it */
-static void tinf_decode_trees(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt)
+static int tinf_decode_trees(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt)
 {
+   /* code lengths for 288 literal/len symbols and 32 dist symbols */
    unsigned char lengths[288+32];
-   unsigned int hlit, hdist, hclen;
+   unsigned int hlit, hdist, hclen, hlimit;
    unsigned int i, num, length;
 
    /* get 5 bits HLIT (257-286) */
@@ -286,53 +334,75 @@ static void tinf_decode_trees(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt)
    tinf_build_tree(lt, lengths, 19);
 
    /* decode code lengths for the dynamic trees */
-   for (num = 0; num < hlit + hdist; )
+   hlimit = hlit + hdist;
+   for (num = 0; num < hlimit; )
    {
       int sym = tinf_decode_symbol(d, lt);
+      unsigned char fill_value = 0;
+      int lbits, lbase = 3;
+
+      /* error decoding */
+      if (sym < 0) return sym;
 
       switch (sym)
       {
       case 16:
          /* copy previous code length 3-6 times (read 2 bits) */
-         {
-            unsigned char prev = lengths[num - 1];
-            for (length = tinf_read_bits(d, 2, 3); length; --length)
-            {
-               lengths[num++] = prev;
-            }
-         }
+         if (num == 0) return TINF_DATA_ERROR;
+         fill_value = lengths[num - 1];
+         lbits = 2;
          break;
       case 17:
          /* repeat code length 0 for 3-10 times (read 3 bits) */
-         for (length = tinf_read_bits(d, 3, 3); length; --length)
-         {
-            lengths[num++] = 0;
-         }
+         lbits = 3;
          break;
       case 18:
          /* repeat code length 0 for 11-138 times (read 7 bits) */
-         for (length = tinf_read_bits(d, 7, 11); length; --length)
-         {
-            lengths[num++] = 0;
-         }
+         lbits = 7;
+         lbase = 11;
          break;
       default:
          /* values 0-15 represent the actual code lengths */
          lengths[num++] = sym;
-         break;
+         /* continue the for loop */
+         continue;
+      }
+
+      /* special code length 16-18 are handled here */
+      length = tinf_read_bits(d, lbits, lbase);
+      if (num + length > hlimit) return TINF_DATA_ERROR;
+      for (; length; --length)
+      {
+         lengths[num++] = fill_value;
       }
    }
 
+   #if UZLIB_CONF_DEBUG_LOG >= 2
+   printf("lit code lengths (%d):", hlit);
+   UZLIB_DUMP_ARRAY("", lengths, hlit);
+   printf("dist code lengths (%d):", hdist);
+   UZLIB_DUMP_ARRAY("", lengths + hlit, hdist);
+   #endif
+
+   #if UZLIB_CONF_PARANOID_CHECKS
+   /* Check that there's "end of block" symbol */
+   if (lengths[256] == 0) {
+      return TINF_DATA_ERROR;
+   }
+   #endif
+
    /* build dynamic trees */
    tinf_build_tree(lt, lengths, hlit);
    tinf_build_tree(dt, lengths + hlit, hdist);
+
+   return TINF_OK;
 }
 
 /* ----------------------------- *
  * -- block inflate functions -- *
  * ----------------------------- */
 
-/* given a stream and two trees, inflate a block of data */
+/* given a stream and two trees, inflate next byte of output */
 static int tinf_inflate_block_data(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt)
 {
     if (d->curlen == 0) {
@@ -341,6 +411,10 @@ static int tinf_inflate_block_data(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt)
         int sym = tinf_decode_symbol(d, lt);
         //printf("huff sym: %02x\n", sym);
 
+        if (d->eof) {
+            return TINF_DATA_ERROR;
+        }
+
         /* literal byte */
         if (sym < 256) {
             TINF_PUT(d, sym);
@@ -354,21 +428,45 @@ static int tinf_inflate_block_data(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt)
 
         /* substring from sliding dictionary */
         sym -= 257;
+        if (sym >= 29) {
+            return TINF_DATA_ERROR;
+        }
+
         /* possibly get more bits from length code */
         d->curlen = tinf_read_bits(d, length_bits[sym], length_base[sym]);
 
         dist = tinf_decode_symbol(d, dt);
+        if (dist >= 30) {
+            return TINF_DATA_ERROR;
+        }
+
         /* possibly get more bits from distance code */
         offs = tinf_read_bits(d, dist_bits[dist], dist_base[dist]);
+
+        /* calculate and validate actual LZ offset to use */
         if (d->dict_ring) {
             if (offs > d->dict_size) {
                 return TINF_DICT_ERROR;
             }
+            /* Note: unlike full-dest-in-memory case below, we don't
+               try to catch offset which points to not yet filled
+               part of the dictionary here. Doing so would require
+               keeping another variable to track "filled in" size
+               of the dictionary. Appearance of such an offset cannot
+               lead to accessing memory outside of the dictionary
+               buffer, and clients which don't want to leak unrelated
+               information, should explicitly initialize dictionary
+               buffer passed to uzlib. */
+
             d->lzOff = d->dict_idx - offs;
             if (d->lzOff < 0) {
                 d->lzOff += d->dict_size;
             }
         } else {
+            /* catch trying to point before the start of dest buffer */
+            if (offs > d->dest - d->destStart) {
+                return TINF_DATA_ERROR;
+            }
             d->lzOff = -offs;
         }
     }
@@ -387,7 +485,7 @@ static int tinf_inflate_block_data(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt)
     return TINF_OK;
 }
 
-/* inflate an uncompressed block of data */
+/* inflate next byte from uncompressed block of data */
 static int tinf_inflate_uncompressed_block(TINF_DATA *d)
 {
     if (d->curlen == 0) {
@@ -440,6 +538,7 @@ void uzlib_init(void)
 /* initialize decompression structure */
 void uzlib_uncompress_init(TINF_DATA *d, void *dict, unsigned int dictLen)
 {
+   d->eof = 0;
    d->bitcount = 0;
    d->bfinal = 0;
    d->btype = -1;
@@ -449,7 +548,7 @@ void uzlib_uncompress_init(TINF_DATA *d, void *dict, unsigned int dictLen)
    d->curlen = 0;
 }
 
-/* inflate next byte of compressed stream */
+/* inflate next output bytes from compressed stream */
 int uzlib_uncompress(TINF_DATA *d)
 {
     do {
@@ -463,14 +562,19 @@ next_blk:
             /* read block type (2 bits) */
             d->btype = tinf_read_bits(d, 2, 0);
 
-            //printf("Started new block: type=%d final=%d\n", d->btype, d->bfinal);
+            #if UZLIB_CONF_DEBUG_LOG >= 1
+            printf("Started new block: type=%d final=%d\n", d->btype, d->bfinal);
+            #endif
 
             if (d->btype == 1) {
                 /* build fixed huffman trees */
                 tinf_build_fixed_trees(&d->ltree, &d->dtree);
             } else if (d->btype == 2) {
                 /* decode trees from stream */
-                tinf_decode_trees(d, &d->ltree, &d->dtree);
+                res = tinf_decode_trees(d, &d->ltree, &d->dtree);
+                if (res != TINF_OK) {
+                    return res;
+                }
             }
         }
 
@@ -483,7 +587,7 @@ next_blk:
             break;
         case 1:
         case 2:
-            /* decompress block with fixed/dyanamic huffman trees */
+            /* decompress block with fixed/dynamic huffman trees */
             /* trees were decoded previously, so it's the same routine for both */
             res = tinf_inflate_block_data(d, &d->ltree, &d->dtree);
             break;
@@ -501,11 +605,13 @@ next_blk:
             return res;
         }
 
-    } while (--d->destSize);
+    } while (d->dest < d->dest_limit);
 
     return TINF_OK;
 }
 
+/* inflate next output bytes from compressed stream, updating
+   checksum, and at the end of stream, verify it */
 int uzlib_uncompress_chksum(TINF_DATA *d)
 {
     int res;

extmod/uzlib/tinfzlib.c

@@ -1,12 +1,12 @@
 /*
- * tinfzlib  -  tiny zlib decompressor
+ * uzlib  -  tiny deflate/inflate library (deflate, gzip, zlib)
  *
  * Copyright (c) 2003 by Joergen Ibsen / Jibz
  * All Rights Reserved
  *
  * http://www.ibsensoftware.com/
  *
- * Copyright (c) 2014-2016 by Paul Sokolovsky
+ * Copyright (c) 2014-2018 by Paul Sokolovsky
  *
  * This software is provided 'as-is', without any express
  * or implied warranty.  In no event will the authors be

extmod/uzlib/uzlib.h

@@ -0,0 +1,169 @@
+/*
+ * uzlib  -  tiny deflate/inflate library (deflate, gzip, zlib)
+ *
+ * Copyright (c) 2003 by Joergen Ibsen / Jibz
+ * All Rights Reserved
+ * http://www.ibsensoftware.com/
+ *
+ * Copyright (c) 2014-2018 by Paul Sokolovsky
+ *
+ * This software is provided 'as-is', without any express
+ * or implied warranty.  In no event will the authors be
+ * held liable for any damages arising from the use of
+ * this software.
+ *
+ * Permission is granted to anyone to use this software
+ * for any purpose, including commercial applications,
+ * and to alter it and redistribute it freely, subject to
+ * the following restrictions:
+ *
+ * 1. The origin of this software must not be
+ *    misrepresented; you must not claim that you
+ *    wrote the original software. If you use this
+ *    software in a product, an acknowledgment in
+ *    the product documentation would be appreciated
+ *    but is not required.
+ *
+ * 2. Altered source versions must be plainly marked
+ *    as such, and must not be misrepresented as
+ *    being the original software.
+ *
+ * 3. This notice may not be removed or altered from
+ *    any source distribution.
+ */
+
+#ifndef UZLIB_H_INCLUDED
+#define UZLIB_H_INCLUDED
+
+#include <stdlib.h>
+#include <stdint.h>
+#include <stdbool.h>
+
+#include "defl_static.h"
+
+#include "uzlib_conf.h"
+#if UZLIB_CONF_DEBUG_LOG
+#include <stdio.h>
+#endif
+
+/* calling convention */
+#ifndef TINFCC
+ #ifdef __WATCOMC__
+  #define TINFCC __cdecl
+ #else
+  #define TINFCC
+ #endif
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* ok status, more data produced */
+#define TINF_OK             0
+/* end of compressed stream reached */
+#define TINF_DONE           1
+#define TINF_DATA_ERROR    (-3)
+#define TINF_CHKSUM_ERROR  (-4)
+#define TINF_DICT_ERROR    (-5)
+
+/* checksum types */
+#define TINF_CHKSUM_NONE  0
+#define TINF_CHKSUM_ADLER 1
+#define TINF_CHKSUM_CRC   2
+
+/* helper macros */
+#define TINF_ARRAY_SIZE(arr) (sizeof(arr) / sizeof(*(arr)))
+
+/* data structures */
+
+typedef struct {
+   unsigned short table[16];  /* table of code length counts */
+   unsigned short trans[288]; /* code -> symbol translation table */
+} TINF_TREE;
+
+struct uzlib_uncomp {
+    /* Pointer to the next byte in the input buffer */
+    const unsigned char *source;
+    /* Pointer to the next byte past the input buffer (source_limit = source + len) */
+    const unsigned char *source_limit;
+    /* If source_limit == NULL, or source >= source_limit, this function
+       will be used to read next byte from source stream. The function may
+       also return -1 in case of EOF (or irrecoverable error). Note that
+       besides returning the next byte, it may also update source and
+       source_limit fields, thus allowing for buffered operation. */
+    int (*source_read_cb)(struct uzlib_uncomp *uncomp);
+
+    unsigned int tag;
+    unsigned int bitcount;
+
+    /* Destination (output) buffer start */
+    unsigned char *dest_start;
+    /* Current pointer in dest buffer */
+    unsigned char *dest;
+    /* Pointer past the end of the dest buffer, similar to source_limit */
+    unsigned char *dest_limit;
+
+    /* Accumulating checksum */
+    unsigned int checksum;
+    char checksum_type;
+    bool eof;
+
+    int btype;
+    int bfinal;
+    unsigned int curlen;
+    int lzOff;
+    unsigned char *dict_ring;
+    unsigned int dict_size;
+    unsigned int dict_idx;
+
+    TINF_TREE ltree; /* dynamic length/symbol tree */
+    TINF_TREE dtree; /* dynamic distance tree */
+};
+
+#include "tinf_compat.h"
+
+#define TINF_PUT(d, c) \
+    { \
+        *d->dest++ = c; \
+        if (d->dict_ring) { d->dict_ring[d->dict_idx++] = c; if (d->dict_idx == d->dict_size) d->dict_idx = 0; } \
+    }
+
+unsigned char TINFCC uzlib_get_byte(TINF_DATA *d);
+
+/* Decompression API */
+
+void TINFCC uzlib_init(void);
+void TINFCC uzlib_uncompress_init(TINF_DATA *d, void *dict, unsigned int dictLen);
+int  TINFCC uzlib_uncompress(TINF_DATA *d);
+int  TINFCC uzlib_uncompress_chksum(TINF_DATA *d);
+
+int TINFCC uzlib_zlib_parse_header(TINF_DATA *d);
+int TINFCC uzlib_gzip_parse_header(TINF_DATA *d);
+
+/* Compression API */
+
+typedef const uint8_t *uzlib_hash_entry_t;
+
+struct uzlib_comp {
+    struct Outbuf out;
+
+    uzlib_hash_entry_t *hash_table;
+    unsigned int hash_bits;
+    unsigned int dict_size;
+};
+
+void TINFCC uzlib_compress(struct uzlib_comp *c, const uint8_t *src, unsigned slen);
+
+/* Checksum API */
+
+/* prev_sum is previous value for incremental computation, 1 initially */
+uint32_t TINFCC uzlib_adler32(const void *data, unsigned int length, uint32_t prev_sum);
+/* crc is previous value for incremental computation, 0xffffffff initially */
+uint32_t TINFCC uzlib_crc32(const void *data, unsigned int length, uint32_t crc);
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif /* UZLIB_H_INCLUDED */

extmod/uzlib/uzlib_conf.h

@@ -0,0 +1,22 @@
+/*
+ * uzlib  -  tiny deflate/inflate library (deflate, gzip, zlib)
+ *
+ * Copyright (c) 2014-2018 by Paul Sokolovsky
+ */
+
+#ifndef UZLIB_CONF_H_INCLUDED
+#define UZLIB_CONF_H_INCLUDED
+
+#ifndef UZLIB_CONF_DEBUG_LOG
+/* Debug logging level 0, 1, 2, etc. */
+#define UZLIB_CONF_DEBUG_LOG 0
+#endif
+
+#ifndef UZLIB_CONF_PARANOID_CHECKS
+/* Perform extra checks on the input stream, even if they aren't proven
+   to be strictly required (== lack of them wasn't proven to lead to
+   crashes). */
+#define UZLIB_CONF_PARANOID_CHECKS 0
+#endif
+
+#endif /* UZLIB_CONF_H_INCLUDED */

extmod/vfs.c

@@ -227,7 +227,7 @@ mp_obj_t mp_vfs_umount(mp_obj_t mnt_in) {
     mp_vfs_mount_t *vfs = NULL;
     size_t mnt_len;
     const char *mnt_str = NULL;
-    if (MP_OBJ_IS_STR(mnt_in)) {
+    if (mp_obj_is_str(mnt_in)) {
         mnt_str = mp_obj_str_get_data(mnt_in, &mnt_len);
     }
     for (mp_vfs_mount_t **vfsp = &MP_STATE_VM(vfs_mount_table); *vfsp != NULL; vfsp = &(*vfsp)->next) {
@@ -270,7 +270,7 @@ mp_obj_t mp_vfs_open(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args)
 
     #if MICROPY_VFS_POSIX
     // If the file is an integer then delegate straight to the POSIX handler
-    if (MP_OBJ_IS_SMALL_INT(args[ARG_file].u_obj)) {
+    if (mp_obj_is_small_int(args[ARG_file].u_obj)) {
         return mp_vfs_posix_file_open(&mp_type_textio, args[ARG_file].u_obj, args[ARG_mode].u_obj);
     }
     #endif

extmod/vfs_fat.c

@@ -39,8 +39,8 @@
 #include "extmod/vfs_fat.h"
 #include "lib/timeutils/timeutils.h"
 
-#if _MAX_SS == _MIN_SS
-#define SECSIZE(fs) (_MIN_SS)
+#if FF_MAX_SS == FF_MIN_SS
+#define SECSIZE(fs) (FF_MIN_SS)
 #else
 #define SECSIZE(fs) ((fs)->ssize)
 #endif
@@ -111,8 +111,11 @@ STATIC mp_obj_t fat_vfs_mkfs(mp_obj_t bdev_in) {
     fs_user_mount_t *vfs = MP_OBJ_TO_PTR(fat_vfs_make_new(&mp_fat_vfs_type, 1, 0, &bdev_in));
 
     // make the filesystem
-    uint8_t working_buf[_MAX_SS];
+    uint8_t working_buf[FF_MAX_SS];
     FRESULT res = f_mkfs(&vfs->fatfs, FM_FAT | FM_SFD, 0, working_buf, sizeof(working_buf));
+    if (res == FR_MKFS_ABORTED) { // Probably doesn't support FAT16
+        res = f_mkfs(&vfs->fatfs, FM_FAT32, 0, working_buf, sizeof(working_buf));
+    }
     if (res != FR_OK) {
         mp_raise_OSError(fresult_to_errno_table[res]);
     }
@@ -363,7 +366,7 @@ STATIC mp_obj_t fat_vfs_statvfs(mp_obj_t vfs_in, mp_obj_t path_in) {
     t->items[6] = MP_OBJ_NEW_SMALL_INT(0); // f_ffree
     t->items[7] = MP_OBJ_NEW_SMALL_INT(0); // f_favail
     t->items[8] = MP_OBJ_NEW_SMALL_INT(0); // f_flags
-    t->items[9] = MP_OBJ_NEW_SMALL_INT(_MAX_LFN); // f_namemax
+    t->items[9] = MP_OBJ_NEW_SMALL_INT(FF_MAX_LFN); // f_namemax
 
     return MP_OBJ_FROM_PTR(t);
 }
@@ -383,7 +386,7 @@ STATIC mp_obj_t vfs_fat_mount(mp_obj_t self_in, mp_obj_t readonly, mp_obj_t mkfs
     // check if we need to make the filesystem
     FRESULT res = (self->flags & FSUSER_NO_FILESYSTEM) ? FR_NO_FILESYSTEM : FR_OK;
     if (res == FR_NO_FILESYSTEM && mp_obj_is_true(mkfs)) {
-        uint8_t working_buf[_MAX_SS];
+        uint8_t working_buf[FF_MAX_SS];
         res = f_mkfs(&self->fatfs, FM_FAT | FM_SFD, 0, working_buf, sizeof(working_buf));
     }
     if (res != FR_OK) {

extmod/vfs_fat_diskio.c

@@ -42,8 +42,8 @@
 #include "lib/oofatfs/diskio.h"
 #include "extmod/vfs_fat.h"
 
-#if _MAX_SS == _MIN_SS
-#define SECSIZE(fs) (_MIN_SS)
+#if FF_MAX_SS == FF_MIN_SS
+#define SECSIZE(fs) (FF_MIN_SS)
 #else
 #define SECSIZE(fs) ((fs)->ssize)
 #endif
@@ -194,7 +194,7 @@ DRESULT disk_ioctl (
             } else {
                 *((WORD*)buff) = mp_obj_get_int(ret);
             }
-            #if _MAX_SS != _MIN_SS
+            #if FF_MAX_SS != FF_MIN_SS
             // need to store ssize because we use it in disk_read/disk_write
             vfs->fatfs.ssize = *((WORD*)buff);
             #endif

extmod/vfs_posix.c

@@ -130,7 +130,7 @@ STATIC mp_obj_t vfs_posix_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode
         && (strchr(mode, 'w') != NULL || strchr(mode, 'a') != NULL || strchr(mode, '+') != NULL)) {
         mp_raise_OSError(MP_EROFS);
     }
-    if (!MP_OBJ_IS_SMALL_INT(path_in)) {
+    if (!mp_obj_is_small_int(path_in)) {
         path_in = vfs_posix_get_path_obj(self, path_in);
     }
     return mp_vfs_posix_file_open(&mp_type_textio, path_in, mode_in);

extmod/vfs_posix_file.c

@@ -94,7 +94,7 @@ mp_obj_t mp_vfs_posix_file_open(const mp_obj_type_t *type, mp_obj_t file_in, mp_
 
     mp_obj_t fid = file_in;
 
-    if (MP_OBJ_IS_SMALL_INT(fid)) {
+    if (mp_obj_is_small_int(fid)) {
         o->fd = MP_OBJ_SMALL_INT_VALUE(fid);
         return MP_OBJ_FROM_PTR(o);
     }

lib/netutils/netutils.h

@@ -29,6 +29,10 @@
 
 #define NETUTILS_IPV4ADDR_BUFSIZE    4
 
+#define NETUTILS_TRACE_IS_TX        (0x0001)
+#define NETUTILS_TRACE_PAYLOAD      (0x0002)
+#define NETUTILS_TRACE_NEWLINE      (0x0004)
+
 typedef enum _netutils_endian_t {
     NETUTILS_LITTLE,
     NETUTILS_BIG,
@@ -47,4 +51,6 @@ void netutils_parse_ipv4_addr(mp_obj_t addr_in, uint8_t *out_ip, netutils_endian
 // puts IP in out_ip (which must take at least IPADDR_BUF_SIZE bytes).
 mp_uint_t netutils_parse_inet_addr(mp_obj_t addr_in, uint8_t *out_ip, netutils_endian_t endian);
 
+void netutils_ethernet_trace(const mp_print_t *print, size_t len, const uint8_t *buf, unsigned int flags);
+
 #endif // MICROPY_INCLUDED_LIB_NETUTILS_NETUTILS_H

lib/netutils/trace.c

@@ -0,0 +1,150 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2019 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "py/mphal.h"
+#include "lib/netutils/netutils.h"
+
+static uint32_t get_be16(const uint8_t *buf) {
+    return buf[0] << 8 | buf[1];
+}
+
+static uint32_t get_be32(const uint8_t *buf) {
+    return buf[0] << 24 | buf[1] << 16 | buf[2] << 8 | buf[3];
+}
+
+static void dump_hex_bytes(const mp_print_t *print, size_t len, const uint8_t *buf) {
+    for (size_t i = 0; i < len; ++i) {
+        mp_printf(print, " %02x", buf[i]);
+    }
+}
+
+static const char *ethertype_str(uint16_t type) {
+    // A value between 0x0000 - 0x05dc (inclusive) indicates a length, not type
+    switch (type) {
+        case 0x0800: return "IPv4";
+        case 0x0806: return "ARP";
+        case 0x86dd: return "IPv6";
+        default: return NULL;
+    }
+}
+
+void netutils_ethernet_trace(const mp_print_t *print, size_t len, const uint8_t *buf, unsigned int flags) {
+    mp_printf(print, "[% 8d] ETH%cX len=%u", mp_hal_ticks_ms(), flags & NETUTILS_TRACE_IS_TX ? 'T' : 'R', len);
+    mp_printf(print, " dst=%02x:%02x:%02x:%02x:%02x:%02x", buf[0], buf[1], buf[2], buf[3], buf[4], buf[5]);
+    mp_printf(print, " src=%02x:%02x:%02x:%02x:%02x:%02x", buf[6], buf[7], buf[8], buf[9], buf[10], buf[11]);
+
+    const char *ethertype = ethertype_str(buf[12] << 8 | buf[13]);
+    if (ethertype) {
+        mp_printf(print, " type=%s", ethertype);
+    } else {
+        mp_printf(print, " type=0x%04x", buf[12] << 8 | buf[13]);
+    }
+    if (len > 14) {
+        len -= 14;
+        buf += 14;
+        if (buf[-2] == 0x08 && buf[-1] == 0x00 && buf[0] == 0x45) {
+            // IPv4 packet
+            len = get_be16(buf + 2);
+            mp_printf(print, " srcip=%u.%u.%u.%u dstip=%u.%u.%u.%u",
+                buf[12], buf[13], buf[14], buf[15],
+                buf[16], buf[17], buf[18], buf[19]);
+            uint8_t prot = buf[9];
+            buf += 20;
+            len -= 20;
+            if (prot == 6) {
+                // TCP packet
+                uint16_t srcport = get_be16(buf);
+                uint16_t dstport = get_be16(buf + 2);
+                uint32_t seqnum = get_be32(buf + 4);
+                uint32_t acknum = get_be32(buf + 8);
+                uint16_t dataoff_flags = get_be16(buf + 12);
+                uint16_t winsz = get_be16(buf + 14);
+                mp_printf(print, " TCP srcport=%u dstport=%u seqnum=%u acknum=%u dataoff=%u flags=%x winsz=%u",
+                    srcport, dstport, (unsigned)seqnum, (unsigned)acknum, dataoff_flags >> 12, dataoff_flags & 0x1ff, winsz);
+                buf += 20;
+                len -= 20;
+                if (dataoff_flags >> 12 > 5) {
+                    mp_printf(print, " opts=");
+                    size_t opts_len = ((dataoff_flags >> 12) - 5) * 4;
+                    dump_hex_bytes(print, opts_len, buf);
+                    buf += opts_len;
+                    len -= opts_len;
+                }
+            } else if (prot == 17) {
+                // UDP packet
+                uint16_t srcport = get_be16(buf);
+                uint16_t dstport = get_be16(buf + 2);
+                mp_printf(print, " UDP srcport=%u dstport=%u", srcport, dstport);
+                len = get_be16(buf + 4);
+                buf += 8;
+                if ((srcport == 67 && dstport == 68) || (srcport == 68 && dstport == 67)) {
+                    // DHCP
+                    if (srcport == 67) {
+                        mp_printf(print, " DHCPS");
+                    } else {
+                        mp_printf(print, " DHCPC");
+                    }
+                    dump_hex_bytes(print, 12 + 16 + 16 + 64, buf);
+                    size_t n = 12 + 16 + 16 + 64 + 128;
+                    len -= n;
+                    buf += n;
+                    mp_printf(print, " opts:");
+                    switch (buf[6]) {
+                        case 1: mp_printf(print, " DISCOVER"); break;
+                        case 2: mp_printf(print, " OFFER"); break;
+                        case 3: mp_printf(print, " REQUEST"); break;
+                        case 4: mp_printf(print, " DECLINE"); break;
+                        case 5: mp_printf(print, " ACK"); break;
+                        case 6: mp_printf(print, " NACK"); break;
+                        case 7: mp_printf(print, " RELEASE"); break;
+                        case 8: mp_printf(print, " INFORM"); break;
+                    }
+                }
+            } else {
+                // Non-UDP packet
+                mp_printf(print, " prot=%u", prot);
+            }
+        } else if (buf[-2] == 0x86 && buf[-1] == 0xdd && (buf[0] >> 4) == 6) {
+            // IPv6 packet
+            uint32_t h = get_be32(buf);
+            uint16_t l = get_be16(buf + 4);
+            mp_printf(print, " tclass=%u flow=%u len=%u nexthdr=%u hoplimit=%u", (unsigned)((h >> 20) & 0xff), (unsigned)(h & 0xfffff), l, buf[6], buf[7]);
+            mp_printf(print, " srcip=");
+            dump_hex_bytes(print, 16, buf + 8);
+            mp_printf(print, " dstip=");
+            dump_hex_bytes(print, 16, buf + 24);
+            buf += 40;
+            len -= 40;
+        }
+        if (flags & NETUTILS_TRACE_PAYLOAD) {
+            mp_printf(print, " data=");
+            dump_hex_bytes(print, len, buf);
+        }
+    }
+    if (flags & NETUTILS_TRACE_NEWLINE) {
+        mp_printf(print, "\n");
+    }
+}

lib/oofatfs/diskio.h

@@ -13,8 +13,6 @@
 extern "C" {
 #endif
 
-
-
 /* Status of Disk Functions */
 typedef BYTE    DSTATUS;
 
@@ -47,11 +45,11 @@ DRESULT disk_ioctl (void *drv, BYTE cmd, void* buff);
 /* Command code for disk_ioctrl fucntion */
 
 /* Generic command (Used by FatFs) */
-#define CTRL_SYNC           0   /* Complete pending write process (needed at _FS_READONLY == 0) */
-#define GET_SECTOR_COUNT    1   /* Get media size (needed at _USE_MKFS == 1) */
-#define GET_SECTOR_SIZE     2   /* Get sector size (needed at _MAX_SS != _MIN_SS) */
-#define GET_BLOCK_SIZE      3   /* Get erase block size (needed at _USE_MKFS == 1) */
-#define CTRL_TRIM           4   /* Inform device that the data on the block of sectors is no longer used (needed at _USE_TRIM == 1) */
+#define CTRL_SYNC           0   /* Complete pending write process (needed at FF_FS_READONLY == 0) */
+#define GET_SECTOR_COUNT    1   /* Get media size (needed at FF_USE_MKFS == 1) */
+#define GET_SECTOR_SIZE     2   /* Get sector size (needed at FF_MAX_SS != FF_MIN_SS) */
+#define GET_BLOCK_SIZE      3   /* Get erase block size (needed at FF_USE_MKFS == 1) */
+#define CTRL_TRIM           4   /* Inform device that the data on the block of sectors is no longer used (needed at FF_USE_TRIM == 1) */
 #define IOCTL_INIT          5
 #define IOCTL_STATUS        6
 

lib/oofatfs/ff.h

@@ -3,10 +3,10 @@
  */
 
 /*----------------------------------------------------------------------------/
-/  FatFs - Generic FAT file system module  R0.12b                             /
+/  FatFs - Generic FAT Filesystem module  R0.13c                              /
 /-----------------------------------------------------------------------------/
 /
-/ Copyright (C) 2016, ChaN, all right reserved.
+/ Copyright (C) 2018, ChaN, all right reserved.
 /
 / FatFs module is an open source software. Redistribution and use of FatFs in
 / source and binary forms, with or without modification, are permitted provided
@@ -19,81 +19,93 @@
 / and any warranties related to this software are DISCLAIMED.
 / The copyright owner or contributors be NOT LIABLE for any damages caused
 / by use of this software.
+/
 /----------------------------------------------------------------------------*/
 
 
-#ifndef _FATFS
-#define _FATFS  68020   /* Revision ID */
+#ifndef FF_DEFINED
+#define FF_DEFINED  86604   /* Revision ID */
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include <stdint.h>
+#include FFCONF_H       /* FatFs configuration options */
 
-/* This type MUST be 8-bit */
-typedef uint8_t BYTE;
-
-/* These types MUST be 16-bit */
-typedef int16_t SHORT;
-typedef uint16_t WORD;
-typedef uint16_t WCHAR;
-
-/* These types MUST be 16-bit or 32-bit */
-typedef int INT;
-typedef unsigned int UINT;
-
-/* These types MUST be 32-bit */
-typedef int32_t LONG;
-typedef uint32_t DWORD;
-
-/* This type MUST be 64-bit (Remove this for C89 compatibility) */
-typedef uint64_t QWORD;
-
-#include FFCONF_H          /* FatFs configuration options */
-
-#if _FATFS != _FFCONF
+#if FF_DEFINED != FFCONF_DEF
 #error Wrong configuration file (ffconf.h).
 #endif
 
 
+/* Integer types used for FatFs API */
+
+#if defined(_WIN32) /* Main development platform */
+#define FF_INTDEF 2
+#include <windows.h>
+typedef unsigned __int64 QWORD;
+#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || defined(__cplusplus)    /* C99 or later */
+#define FF_INTDEF 2
+#include <stdint.h>
+typedef unsigned int    UINT;   /* int must be 16-bit or 32-bit */
+typedef unsigned char   BYTE;   /* char must be 8-bit */
+typedef uint16_t        WORD;   /* 16-bit unsigned integer */
+typedef uint16_t        WCHAR;  /* 16-bit unsigned integer */
+typedef uint32_t        DWORD;  /* 32-bit unsigned integer */
+typedef uint64_t        QWORD;  /* 64-bit unsigned integer */
+#else   /* Earlier than C99 */
+#define FF_INTDEF 1
+typedef unsigned int    UINT;   /* int must be 16-bit or 32-bit */
+typedef unsigned char   BYTE;   /* char must be 8-bit */
+typedef unsigned short  WORD;   /* 16-bit unsigned integer */
+typedef unsigned short  WCHAR;  /* 16-bit unsigned integer */
+typedef unsigned long   DWORD;  /* 32-bit unsigned integer */
+#endif
+
 
 /* Definitions of volume management */
 
-#if _MULTI_PARTITION            /* Multiple partition configuration */
-#define LD2PT(fs) (fs->part)    /* Get partition index */
-#else                           /* Single partition configuration */
-#define LD2PT(fs) 0             /* Find first valid partition or in SFD */
+#if FF_STR_VOLUME_ID
+#ifndef FF_VOLUME_STRS
+extern const char* VolumeStr[FF_VOLUMES];   /* User defied volume ID */
+#endif
 #endif
 
 
 
 /* Type of path name strings on FatFs API */
 
-#if _LFN_UNICODE            /* Unicode (UTF-16) string */
-#if _USE_LFN == 0
-#error _LFN_UNICODE must be 0 at non-LFN cfg.
-#endif
 #ifndef _INC_TCHAR
+#define _INC_TCHAR
+
+#if FF_USE_LFN && FF_LFN_UNICODE == 1   /* Unicode in UTF-16 encoding */
 typedef WCHAR TCHAR;
 #define _T(x) L ## x
 #define _TEXT(x) L ## x
-#endif
-#else                       /* ANSI/OEM string */
-#ifndef _INC_TCHAR
+#elif FF_USE_LFN && FF_LFN_UNICODE == 2 /* Unicode in UTF-8 encoding */
+typedef char TCHAR;
+#define _T(x) u8 ## x
+#define _TEXT(x) u8 ## x
+#elif FF_USE_LFN && FF_LFN_UNICODE == 3 /* Unicode in UTF-32 encoding */
+typedef DWORD TCHAR;
+#define _T(x) U ## x
+#define _TEXT(x) U ## x
+#elif FF_USE_LFN && (FF_LFN_UNICODE < 0 || FF_LFN_UNICODE > 3)
+#error Wrong FF_LFN_UNICODE setting
+#else                                   /* ANSI/OEM code in SBCS/DBCS */
 typedef char TCHAR;
 #define _T(x) x
 #define _TEXT(x) x
 #endif
+
 #endif
 
 
 
 /* Type of file size variables */
 
-#if _FS_EXFAT
-#if _USE_LFN == 0
-#error LFN must be enabled when enable exFAT
+#if FF_FS_EXFAT
+#if FF_INTDEF != 2
+#error exFAT feature wants C99 or later
 #endif
 typedef QWORD FSIZE_t;
 #else
@@ -102,39 +114,39 @@ typedef DWORD FSIZE_t;
 
 
 
-/* File system object structure (FATFS) */
+/* Filesystem object structure (FATFS) */
 
 typedef struct {
     void    *drv;           // block device underlying this filesystem
-#if _MULTI_PARTITION        /* Multiple partition configuration */
+#if FF_MULTI_PARTITION      /* Multiple partition configuration */
     BYTE    part;           // Partition: 0:Auto detect, 1-4:Forced partition
 #endif
-    BYTE    fs_type;        /* File system type (0:N/A) */
+    BYTE    fs_type;        /* Filesystem type (0:not mounted) */
     BYTE    n_fats;         /* Number of FATs (1 or 2) */
     BYTE    wflag;          /* win[] flag (b0:dirty) */
     BYTE    fsi_flag;       /* FSINFO flags (b7:disabled, b0:dirty) */
-    WORD    id;             /* File system mount ID */
+    WORD    id;             /* Volume mount ID */
     WORD    n_rootdir;      /* Number of root directory entries (FAT12/16) */
     WORD    csize;          /* Cluster size [sectors] */
-#if _MAX_SS != _MIN_SS
+#if FF_MAX_SS != FF_MIN_SS
     WORD    ssize;          /* Sector size (512, 1024, 2048 or 4096) */
 #endif
-#if _USE_LFN != 0
+#if FF_USE_LFN
     WCHAR*  lfnbuf;         /* LFN working buffer */
 #endif
-#if _FS_EXFAT
-    BYTE*   dirbuf;         /* Directory entry block scratchpad buffer */
+#if FF_FS_EXFAT
+    BYTE*   dirbuf;         /* Directory entry block scratchpad buffer for exFAT */
 #endif
-#if _FS_REENTRANT
-    _SYNC_t sobj;           /* Identifier of sync object */
+#if FF_FS_REENTRANT
+    FF_SYNC_t   sobj;       /* Identifier of sync object */
 #endif
-#if !_FS_READONLY
+#if !FF_FS_READONLY
     DWORD   last_clst;      /* Last allocated cluster */
     DWORD   free_clst;      /* Number of free clusters */
 #endif
-#if _FS_RPATH != 0
+#if FF_FS_RPATH
     DWORD   cdir;           /* Current directory start cluster (0:root) */
-#if _FS_EXFAT
+#if FF_FS_EXFAT
     DWORD   cdc_scl;        /* Containing directory start cluster (invalid when cdir is 0) */
     DWORD   cdc_size;       /* b31-b8:Size of containing directory, b7-b0: Chain status */
     DWORD   cdc_ofs;        /* Offset in the containing directory (invalid when cdir is 0) */
@@ -146,52 +158,56 @@ typedef struct {
     DWORD   fatbase;        /* FAT base sector */
     DWORD   dirbase;        /* Root directory base sector/cluster */
     DWORD   database;       /* Data base sector */
+#if FF_FS_EXFAT
+    DWORD   bitbase;        /* Allocation bitmap base sector */
+#endif
     DWORD   winsect;        /* Current sector appearing in the win[] */
-    BYTE    win[_MAX_SS];   /* Disk access window for Directory, FAT (and file data at tiny cfg) */
+    BYTE    win[FF_MAX_SS]; /* Disk access window for Directory, FAT (and file data at tiny cfg) */
 } FATFS;
 
 
 
-/* Object ID and allocation information (_FDID) */
+/* Object ID and allocation information (FFOBJID) */
 
 typedef struct {
-    FATFS*  fs;         /* Pointer to the owner file system object */
-    WORD    id;         /* Owner file system mount ID */
-    BYTE    attr;       /* Object attribute */
-    BYTE    stat;       /* Object chain status (b1-0: =0:not contiguous, =2:contiguous (no data on FAT), =3:got flagmented, b2:sub-directory stretched) */
-    DWORD   sclust;     /* Object start cluster (0:no cluster or root directory) */
-    FSIZE_t objsize;    /* Object size (valid when sclust != 0) */
-#if _FS_EXFAT
-    DWORD   n_cont;     /* Size of coutiguous part, clusters - 1 (valid when stat == 3) */
-    DWORD   c_scl;      /* Containing directory start cluster (valid when sclust != 0) */
-    DWORD   c_size;     /* b31-b8:Size of containing directory, b7-b0: Chain status (valid when c_scl != 0) */
-    DWORD   c_ofs;      /* Offset in the containing directory (valid when sclust != 0) */
+    FATFS*  fs;             /* Pointer to the hosting volume of this object */
+    WORD    id;             /* Hosting volume mount ID */
+    BYTE    attr;           /* Object attribute */
+    BYTE    stat;           /* Object chain status (b1-0: =0:not contiguous, =2:contiguous, =3:fragmented in this session, b2:sub-directory stretched) */
+    DWORD   sclust;         /* Object data start cluster (0:no cluster or root directory) */
+    FSIZE_t objsize;        /* Object size (valid when sclust != 0) */
+#if FF_FS_EXFAT
+    DWORD   n_cont;         /* Size of first fragment - 1 (valid when stat == 3) */
+    DWORD   n_frag;         /* Size of last fragment needs to be written to FAT (valid when not zero) */
+    DWORD   c_scl;          /* Containing directory start cluster (valid when sclust != 0) */
+    DWORD   c_size;         /* b31-b8:Size of containing directory, b7-b0: Chain status (valid when c_scl != 0) */
+    DWORD   c_ofs;          /* Offset in the containing directory (valid when file object and sclust != 0) */
 #endif
-#if _FS_LOCK != 0
-    UINT    lockid;     /* File lock ID origin from 1 (index of file semaphore table Files[]) */
+#if FF_FS_LOCK
+    UINT    lockid;         /* File lock ID origin from 1 (index of file semaphore table Files[]) */
 #endif
-} _FDID;
+} FFOBJID;
 
 
 
 /* File object structure (FIL) */
 
 typedef struct {
-    _FDID   obj;            /* Object identifier (must be the 1st member to detect invalid object pointer) */
+    FFOBJID obj;            /* Object identifier (must be the 1st member to detect invalid object pointer) */
     BYTE    flag;           /* File status flags */
     BYTE    err;            /* Abort flag (error code) */
     FSIZE_t fptr;           /* File read/write pointer (Zeroed on file open) */
-    DWORD   clust;          /* Current cluster of fpter (invalid when fprt is 0) */
+    DWORD   clust;          /* Current cluster of fpter (invalid when fptr is 0) */
     DWORD   sect;           /* Sector number appearing in buf[] (0:invalid) */
-#if !_FS_READONLY
-    DWORD   dir_sect;       /* Sector number containing the directory entry */
-    BYTE*   dir_ptr;        /* Pointer to the directory entry in the win[] */
+#if !FF_FS_READONLY
+    DWORD   dir_sect;       /* Sector number containing the directory entry (not used at exFAT) */
+    BYTE*   dir_ptr;        /* Pointer to the directory entry in the win[] (not used at exFAT) */
 #endif
-#if _USE_FASTSEEK
+#if FF_USE_FASTSEEK
     DWORD*  cltbl;          /* Pointer to the cluster link map table (nulled on open, set by application) */
 #endif
-#if !_FS_TINY
-    BYTE    buf[_MAX_SS];   /* File private data read/write window */
+#if !FF_FS_TINY
+    BYTE    buf[FF_MAX_SS]; /* File private data read/write window */
 #endif
 } FIL;
 
@@ -200,16 +216,16 @@ typedef struct {
 /* Directory object structure (FF_DIR) */
 
 typedef struct {
-    _FDID   obj;            /* Object identifier */
+    FFOBJID obj;            /* Object identifier */
     DWORD   dptr;           /* Current read/write offset */
     DWORD   clust;          /* Current cluster */
-    DWORD   sect;           /* Current sector */
+    DWORD   sect;           /* Current sector (0:Read operation has terminated) */
     BYTE*   dir;            /* Pointer to the directory item in the win[] */
     BYTE    fn[12];         /* SFN (in/out) {body[8],ext[3],status[1]} */
-#if _USE_LFN != 0
+#if FF_USE_LFN
     DWORD   blk_ofs;        /* Offset of current entry block being processed (0xFFFFFFFF:Invalid) */
 #endif
-#if _USE_FIND
+#if FF_USE_FIND
     const TCHAR* pat;       /* Pointer to the name matching pattern */
 #endif
 } FF_DIR;
@@ -223,11 +239,11 @@ typedef struct {
     WORD    fdate;          /* Modified date */
     WORD    ftime;          /* Modified time */
     BYTE    fattrib;        /* File attribute */
-#if _USE_LFN != 0
-    TCHAR   altname[13];            /* Altenative file name */
-    TCHAR   fname[_MAX_LFN + 1];    /* Primary file name */
+#if FF_USE_LFN
+    TCHAR   altname[FF_SFN_BUF + 1];/* Altenative file name */
+    TCHAR   fname[FF_LFN_BUF + 1];  /* Primary file name */
 #else
-    TCHAR   fname[13];      /* File name */
+    TCHAR   fname[12 + 1];  /* File name */
 #endif
 } FILINFO;
 
@@ -254,7 +270,7 @@ typedef enum {
     FR_TIMEOUT,             /* (15) Could not get a grant to access the volume within defined period */
     FR_LOCKED,              /* (16) The operation is rejected according to the file sharing policy */
     FR_NOT_ENOUGH_CORE,     /* (17) LFN working buffer could not be allocated */
-    FR_TOO_MANY_OPEN_FILES, /* (18) Number of open files > _FS_LOCK */
+    FR_TOO_MANY_OPEN_FILES, /* (18) Number of open files > FF_FS_LOCK */
     FR_INVALID_PARAMETER    /* (19) Given parameter is invalid */
 } FRESULT;
 
@@ -292,6 +308,7 @@ FRESULT f_mount (FATFS* fs);                                        /* Mount/Unm
 FRESULT f_umount (FATFS* fs);                                       /* Unmount a logical drive */
 FRESULT f_mkfs (FATFS *fs, BYTE opt, DWORD au, void* work, UINT len); /* Create a FAT volume */
 FRESULT f_fdisk (void *pdrv, const DWORD* szt, void* work);         /* Divide a physical drive into some partitions */
+FRESULT f_setcp (WORD cp);                                          /* Set current code page */
 
 #define f_eof(fp) ((int)((fp)->fptr == (fp)->obj.objsize))
 #define f_error(fp) ((fp)->err)
@@ -299,6 +316,8 @@ FRESULT f_fdisk (void *pdrv, const DWORD* szt, void* work);         /* Divide a
 #define f_size(fp) ((fp)->obj.objsize)
 #define f_rewind(fp) f_lseek((fp), 0)
 #define f_rewinddir(dp) f_readdir((dp), 0)
+#define f_rmdir(path) f_unlink(path)
+#define f_unmount(path) f_mount(0, path, 0)
 
 #ifndef EOF
 #define EOF (-1)
@@ -311,26 +330,27 @@ FRESULT f_fdisk (void *pdrv, const DWORD* szt, void* work);         /* Divide a
 /* Additional user defined functions                            */
 
 /* RTC function */
-#if !_FS_READONLY && !_FS_NORTC
+#if !FF_FS_READONLY && !FF_FS_NORTC
 DWORD get_fattime (void);
 #endif
 
-/* Unicode support functions */
-#if _USE_LFN != 0                       /* Unicode - OEM code conversion */
-WCHAR ff_convert (WCHAR chr, UINT dir); /* OEM-Unicode bidirectional conversion */
-WCHAR ff_wtoupper (WCHAR chr);          /* Unicode upper-case conversion */
-#if _USE_LFN == 3                       /* Memory functions */
+/* LFN support functions */
+#if FF_USE_LFN >= 1                     /* Code conversion (defined in unicode.c) */
+WCHAR ff_oem2uni (WCHAR oem, WORD cp);  /* OEM code to Unicode conversion */
+WCHAR ff_uni2oem (DWORD uni, WORD cp);  /* Unicode to OEM code conversion */
+DWORD ff_wtoupper (DWORD uni);          /* Unicode upper-case conversion */
+#endif
+#if FF_USE_LFN == 3                     /* Dynamic memory allocation */
 void* ff_memalloc (UINT msize);         /* Allocate memory block */
 void ff_memfree (void* mblock);         /* Free memory block */
 #endif
-#endif
 
 /* Sync functions */
-#if _FS_REENTRANT
-int ff_cre_syncobj (FATFS *fatfs, _SYNC_t* sobj); /* Create a sync object */
-int ff_req_grant (_SYNC_t sobj);                /* Lock sync object */
-void ff_rel_grant (_SYNC_t sobj);               /* Unlock sync object */
-int ff_del_syncobj (_SYNC_t sobj);              /* Delete a sync object */
+#if FF_FS_REENTRANT
+int ff_cre_syncobj (FATFS *fatfs, FF_SYNC_t* sobj); /* Create a sync object */
+int ff_req_grant (FF_SYNC_t sobj);      /* Lock sync object */
+void ff_rel_grant (FF_SYNC_t sobj);     /* Unlock sync object */
+int ff_del_syncobj (FF_SYNC_t sobj);    /* Delete a sync object */
 #endif
 
 
@@ -377,4 +397,4 @@ int ff_del_syncobj (_SYNC_t sobj);              /* Delete a sync object */
 }
 #endif
 
-#endif /* _FATFS */
+#endif /* FF_DEFINED */

lib/oofatfs/ffconf.h

@@ -2,11 +2,11 @@
  * This file is part of the MicroPython project, http://micropython.org/
  *
  * Original file from:
- * FatFs - FAT file system module configuration file R0.12a (C)ChaN, 2016
+ * FatFs - FAT file system module configuration file R0.13c (C)ChaN, 2018
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2013-2017 Damien P. George
+ * Copyright (c) 2013-2019 Damien P. George
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -30,73 +30,72 @@
 #include "py/mpconfig.h"
 
 /*---------------------------------------------------------------------------/
-/  FatFs - FAT file system module configuration file
+/  FatFs Functional Configurations
 /---------------------------------------------------------------------------*/
 
-#define _FFCONF 68020   /* Revision ID */
+#define FFCONF_DEF  86604   /* Revision ID */
 
 /*---------------------------------------------------------------------------/
 / Function Configurations
 /---------------------------------------------------------------------------*/
 
-#define _FS_READONLY    0
+#define FF_FS_READONLY  0
 /* This option switches read-only configuration. (0:Read/Write or 1:Read-only)
 /  Read-only configuration removes writing API functions, f_write(), f_sync(),
 /  f_unlink(), f_mkdir(), f_chmod(), f_rename(), f_truncate(), f_getfree()
 /  and optional writing functions as well. */
 
 
-#define _FS_MINIMIZE    0
+#define FF_FS_MINIMIZE  0
 /* This option defines minimization level to remove some basic API functions.
 /
-/   0: All basic functions are enabled.
+/   0: Basic functions are fully enabled.
 /   1: f_stat(), f_getfree(), f_unlink(), f_mkdir(), f_truncate() and f_rename()
 /      are removed.
 /   2: f_opendir(), f_readdir() and f_closedir() are removed in addition to 1.
 /   3: f_lseek() function is removed in addition to 2. */
 
 
-#define _USE_STRFUNC    0
-/* This option switches string functions, f_gets(), f_putc(), f_puts() and
-/  f_printf().
+#define FF_USE_STRFUNC  0
+/* This option switches string functions, f_gets(), f_putc(), f_puts() and f_printf().
 /
 /  0: Disable string functions.
 /  1: Enable without LF-CRLF conversion.
 /  2: Enable with LF-CRLF conversion. */
 
 
-#define _USE_FIND       0
+#define FF_USE_FIND     0
 /* This option switches filtered directory read functions, f_findfirst() and
 /  f_findnext(). (0:Disable, 1:Enable 2:Enable with matching altname[] too) */
 
 
-#define _USE_MKFS       1
+#define FF_USE_MKFS     1
 /* This option switches f_mkfs() function. (0:Disable or 1:Enable) */
 
 
-#define _USE_FASTSEEK   0
+#define FF_USE_FASTSEEK 0
 /* This option switches fast seek function. (0:Disable or 1:Enable) */
 
 
-#define _USE_EXPAND     0
+#define FF_USE_EXPAND   0
 /* This option switches f_expand function. (0:Disable or 1:Enable) */
 
 
-#define _USE_CHMOD      1
+#define FF_USE_CHMOD    1
 /* This option switches attribute manipulation functions, f_chmod() and f_utime().
-/  (0:Disable or 1:Enable) Also _FS_READONLY needs to be 0 to enable this option. */
+/  (0:Disable or 1:Enable) Also FF_FS_READONLY needs to be 0 to enable this option. */
 
 
 #ifdef MICROPY_FATFS_USE_LABEL
-#define _USE_LABEL      (MICROPY_FATFS_USE_LABEL)
+#define FF_USE_LABEL    (MICROPY_FATFS_USE_LABEL)
 #else
-#define _USE_LABEL      0
+#define FF_USE_LABEL    0
 #endif
 /* This option switches volume label functions, f_getlabel() and f_setlabel().
 /  (0:Disable or 1:Enable) */
 
 
-#define _USE_FORWARD    0
+#define FF_USE_FORWARD  0
 /* This option switches f_forward() function. (0:Disable or 1:Enable) */
 
 
@@ -105,14 +104,13 @@
 /---------------------------------------------------------------------------*/
 
 #ifdef MICROPY_FATFS_LFN_CODE_PAGE
-#define _CODE_PAGE  (MICROPY_FATFS_LFN_CODE_PAGE)
+#define FF_CODE_PAGE MICROPY_FATFS_LFN_CODE_PAGE
 #else
-#define _CODE_PAGE  1
+#define FF_CODE_PAGE 437
 #endif
 /* This option specifies the OEM code page to be used on the target system.
-/  Incorrect setting of the code page can cause a file open failure.
+/  Incorrect code page setting can cause a file open failure.
 /
-/   1   - ASCII (No extended character. Non-LFN cfg. only)
 /   437 - U.S.
 /   720 - Arabic
 /   737 - Greek
@@ -134,59 +132,77 @@
 /   936 - Simplified Chinese (DBCS)
 /   949 - Korean (DBCS)
 /   950 - Traditional Chinese (DBCS)
+/     0 - Include all code pages above and configured by f_setcp()
 */
 
 
 #ifdef MICROPY_FATFS_ENABLE_LFN
-#define _USE_LFN    (MICROPY_FATFS_ENABLE_LFN)
+#define FF_USE_LFN  (MICROPY_FATFS_ENABLE_LFN)
 #else
-#define _USE_LFN    0
+#define FF_USE_LFN  0
 #endif
 #ifdef MICROPY_FATFS_MAX_LFN
-#define _MAX_LFN    (MICROPY_FATFS_MAX_LFN)
+#define FF_MAX_LFN  (MICROPY_FATFS_MAX_LFN)
 #else
-#define _MAX_LFN    255
+#define FF_MAX_LFN  255
 #endif
-/* The _USE_LFN switches the support of long file name (LFN).
+/* The FF_USE_LFN switches the support for LFN (long file name).
 /
-/   0: Disable support of LFN. _MAX_LFN has no effect.
+/   0: Disable LFN. FF_MAX_LFN has no effect.
 /   1: Enable LFN with static working buffer on the BSS. Always NOT thread-safe.
 /   2: Enable LFN with dynamic working buffer on the STACK.
 /   3: Enable LFN with dynamic working buffer on the HEAP.
 /
-/  To enable the LFN, Unicode handling functions (option/unicode.c) must be added
-/  to the project. The working buffer occupies (_MAX_LFN + 1) * 2 bytes and
-/  additional 608 bytes at exFAT enabled. _MAX_LFN can be in range from 12 to 255.
-/  It should be set 255 to support full featured LFN operations.
+/  To enable the LFN, ffunicode.c needs to be added to the project. The LFN function
+/  requiers certain internal working buffer occupies (FF_MAX_LFN + 1) * 2 bytes and
+/  additional (FF_MAX_LFN + 44) / 15 * 32 bytes when exFAT is enabled.
+/  The FF_MAX_LFN defines size of the working buffer in UTF-16 code unit and it can
+/  be in range of 12 to 255. It is recommended to be set 255 to fully support LFN
+/  specification.
 /  When use stack for the working buffer, take care on stack overflow. When use heap
 /  memory for the working buffer, memory management functions, ff_memalloc() and
-/  ff_memfree(), must be added to the project. */
+/  ff_memfree() in ffsystem.c, need to be added to the project. */
 
 
-#define _LFN_UNICODE    0
-/* This option switches character encoding on the API. (0:ANSI/OEM or 1:UTF-16)
-/  To use Unicode string for the path name, enable LFN and set _LFN_UNICODE = 1.
-/  This option also affects behavior of string I/O functions. */
-
-
-#define _STRF_ENCODE    3
-/* When _LFN_UNICODE == 1, this option selects the character encoding ON THE FILE to
-/  be read/written via string I/O functions, f_gets(), f_putc(), f_puts and f_printf().
+#define FF_LFN_UNICODE  0
+/* This option switches the character encoding on the API when LFN is enabled.
 /
-/  0: ANSI/OEM
-/  1: UTF-16LE
-/  2: UTF-16BE
-/  3: UTF-8
+/   0: ANSI/OEM in current CP (TCHAR = char)
+/   1: Unicode in UTF-16 (TCHAR = WCHAR)
+/   2: Unicode in UTF-8 (TCHAR = char)
+/   3: Unicode in UTF-32 (TCHAR = DWORD)
 /
-/  This option has no effect when _LFN_UNICODE == 0. */
+/  Also behavior of string I/O functions will be affected by this option.
+/  When LFN is not enabled, this option has no effect. */
+
+
+#define FF_LFN_BUF      255
+#define FF_SFN_BUF      12
+/* This set of options defines size of file name members in the FILINFO structure
+/  which is used to read out directory items. These values should be suffcient for
+/  the file names to read. The maximum possible length of the read file name depends
+/  on character encoding. When LFN is not enabled, these options have no effect. */
+
+
+#define FF_STRF_ENCODE  3
+/* When FF_LFN_UNICODE >= 1 with LFN enabled, string I/O functions, f_gets(),
+/  f_putc(), f_puts and f_printf() convert the character encoding in it.
+/  This option selects assumption of character encoding ON THE FILE to be
+/  read/written via those functions.
+/
+/   0: ANSI/OEM in current CP
+/   1: Unicode in UTF-16LE
+/   2: Unicode in UTF-16BE
+/   3: Unicode in UTF-8
+*/
 
 
 #ifdef MICROPY_FATFS_RPATH
-#define _FS_RPATH   (MICROPY_FATFS_RPATH)
+#define FF_FS_RPATH (MICROPY_FATFS_RPATH)
 #else
-#define _FS_RPATH   0
+#define FF_FS_RPATH 0
 #endif
-/* This option configures support of relative path.
+/* This option configures support for relative path.
 /
 /   0: Disable relative path and remove related functions.
 /   1: Enable relative path. f_chdir() and f_chdrive() are available.
@@ -198,53 +214,58 @@
 / Drive/Volume Configurations
 /---------------------------------------------------------------------------*/
 
-#define _VOLUMES    1
-/* Number of volumes (logical drives) to be used. */
+#define FF_VOLUMES      1
+/* Number of volumes (logical drives) to be used. (1-10) */
 
 
-#define _STR_VOLUME_ID  0
-#define _VOLUME_STRS    "RAM","NAND","CF","SD","SD2","USB","USB2","USB3"
-/* _STR_VOLUME_ID switches string support of volume ID.
-/  When _STR_VOLUME_ID is set to 1, also pre-defined strings can be used as drive
-/  number in the path name. _VOLUME_STRS defines the drive ID strings for each
-/  logical drives. Number of items must be equal to _VOLUMES. Valid characters for
-/  the drive ID strings are: A-Z and 0-9. */
+#define FF_STR_VOLUME_ID    0
+#define FF_VOLUME_STRS      "RAM","NAND","CF","SD","SD2","USB","USB2","USB3"
+/* FF_STR_VOLUME_ID switches support for volume ID in arbitrary strings.
+/  When FF_STR_VOLUME_ID is set to 1 or 2, arbitrary strings can be used as drive
+/  number in the path name. FF_VOLUME_STRS defines the volume ID strings for each
+/  logical drives. Number of items must not be less than FF_VOLUMES. Valid
+/  characters for the volume ID strings are A-Z, a-z and 0-9, however, they are
+/  compared in case-insensitive. If FF_STR_VOLUME_ID >= 1 and FF_VOLUME_STRS is
+/  not defined, a user defined volume string table needs to be defined as:
+/
+/  const char* VolumeStr[FF_VOLUMES] = {"ram","flash","sd","usb",...
+*/
 
 
 #ifdef MICROPY_FATFS_MULTI_PARTITION
-#define _MULTI_PARTITION    (MICROPY_FATFS_MULTI_PARTITION)
+#define FF_MULTI_PARTITION  (MICROPY_FATFS_MULTI_PARTITION)
 #else
-#define _MULTI_PARTITION    0
+#define FF_MULTI_PARTITION  0
 #endif
-/* This option switches support of multi-partition on a physical drive.
+/* This option switches support for multiple volumes on the physical drive.
 /  By default (0), each logical drive number is bound to the same physical drive
 /  number and only an FAT volume found on the physical drive will be mounted.
-/  When multi-partition is enabled (1), each logical drive number can be bound to
+/  When this function is enabled (1), each logical drive number can be bound to
 /  arbitrary physical drive and partition listed in the VolToPart[]. Also f_fdisk()
 /  funciton will be available. */
 
 
-#define _MIN_SS     512
+#define FF_MIN_SS   512
 #ifdef MICROPY_FATFS_MAX_SS
-#define _MAX_SS     (MICROPY_FATFS_MAX_SS)
+#define FF_MAX_SS   (MICROPY_FATFS_MAX_SS)
 #else
-#define _MAX_SS     512
+#define FF_MAX_SS   512
 #endif
-/* These options configure the range of sector size to be supported. (512, 1024,
-/  2048 or 4096) Always set both 512 for most systems, all type of memory cards and
+/* This set of options configures the range of sector size to be supported. (512,
+/  1024, 2048 or 4096) Always set both 512 for most systems, generic memory card and
 /  harddisk. But a larger value may be required for on-board flash memory and some
-/  type of optical media. When _MAX_SS is larger than _MIN_SS, FatFs is configured
-/  to variable sector size and GET_SECTOR_SIZE command must be implemented to the
-/  disk_ioctl() function. */
+/  type of optical media. When FF_MAX_SS is larger than FF_MIN_SS, FatFs is configured
+/  for variable sector size mode and disk_ioctl() function needs to implement
+/  GET_SECTOR_SIZE command. */
 
 
-#define _USE_TRIM   0
-/* This option switches support of ATA-TRIM. (0:Disable or 1:Enable)
+#define FF_USE_TRIM     0
+/* This option switches support for ATA-TRIM. (0:Disable or 1:Enable)
 /  To enable Trim function, also CTRL_TRIM command should be implemented to the
 /  disk_ioctl() function. */
 
 
-#define _FS_NOFSINFO    0
+#define FF_FS_NOFSINFO  0
 /* If you need to know correct free space on the FAT32 volume, set bit 0 of this
 /  option, and f_getfree() function at first time after volume mount will force
 /  a full FAT scan. Bit 1 controls the use of last allocated cluster number.
@@ -261,44 +282,44 @@
 / System Configurations
 /---------------------------------------------------------------------------*/
 
-#define _FS_TINY    1
+#define FF_FS_TINY      1
 /* This option switches tiny buffer configuration. (0:Normal or 1:Tiny)
-/  At the tiny configuration, size of file object (FIL) is reduced _MAX_SS bytes.
+/  At the tiny configuration, size of file object (FIL) is shrinked FF_MAX_SS bytes.
 /  Instead of private sector buffer eliminated from the file object, common sector
-/  buffer in the file system object (FATFS) is used for the file data transfer. */
+/  buffer in the filesystem object (FATFS) is used for the file data transfer. */
 
 
 #ifdef MICROPY_FATFS_EXFAT
-#define _FS_EXFAT   (MICROPY_FATFS_EXFAT)
+#define FF_FS_EXFAT (MICROPY_FATFS_EXFAT)
 #else
-#define _FS_EXFAT   0
+#define FF_FS_EXFAT 0
 #endif
-/* This option switches support of exFAT file system. (0:Disable or 1:Enable)
-/  When enable exFAT, also LFN needs to be enabled. (_USE_LFN >= 1)
-/  Note that enabling exFAT discards C89 compatibility. */
+/* This option switches support for exFAT filesystem. (0:Disable or 1:Enable)
+/  To enable exFAT, also LFN needs to be enabled. (FF_USE_LFN >= 1)
+/  Note that enabling exFAT discards ANSI C (C89) compatibility. */
 
 
 #ifdef MICROPY_FATFS_NORTC
-#define _FS_NORTC   (MICROPY_FATFS_NORTC)
+#define FF_FS_NORTC (MICROPY_FATFS_NORTC)
 #else
-#define _FS_NORTC   0
+#define FF_FS_NORTC 0
 #endif
-#define _NORTC_MON  1
-#define _NORTC_MDAY 1
-#define _NORTC_YEAR 2016
-/* The option _FS_NORTC switches timestamp functiton. If the system does not have
-/  any RTC function or valid timestamp is not needed, set _FS_NORTC = 1 to disable
-/  the timestamp function. All objects modified by FatFs will have a fixed timestamp
-/  defined by _NORTC_MON, _NORTC_MDAY and _NORTC_YEAR in local time.
-/  To enable timestamp function (_FS_NORTC = 0), get_fattime() function need to be
-/  added to the project to get current time form real-time clock. _NORTC_MON,
-/  _NORTC_MDAY and _NORTC_YEAR have no effect.
-/  These options have no effect at read-only configuration (_FS_READONLY = 1). */
+#define FF_NORTC_MON    1
+#define FF_NORTC_MDAY   1
+#define FF_NORTC_YEAR   2018
+/* The option FF_FS_NORTC switches timestamp functiton. If the system does not have
+/  any RTC function or valid timestamp is not needed, set FF_FS_NORTC = 1 to disable
+/  the timestamp function. Every object modified by FatFs will have a fixed timestamp
+/  defined by FF_NORTC_MON, FF_NORTC_MDAY and FF_NORTC_YEAR in local time.
+/  To enable timestamp function (FF_FS_NORTC = 0), get_fattime() function need to be
+/  added to the project to read current time form real-time clock. FF_NORTC_MON,
+/  FF_NORTC_MDAY and FF_NORTC_YEAR have no effect.
+/  These options have no effect at read-only configuration (FF_FS_READONLY = 1). */
 
 
-#define _FS_LOCK    0
-/* The option _FS_LOCK switches file lock function to control duplicated file open
-/  and illegal operation to open objects. This option must be 0 when _FS_READONLY
+#define FF_FS_LOCK      0
+/* The option FF_FS_LOCK switches file lock function to control duplicated file open
+/  and illegal operation to open objects. This option must be 0 when FF_FS_READONLY
 /  is 1.
 /
 /  0:  Disable file lock function. To avoid volume corruption, application program
@@ -309,41 +330,40 @@
 
 
 #ifdef MICROPY_FATFS_REENTRANT
-#define _FS_REENTRANT   (MICROPY_FATFS_REENTRANT)
+#define FF_FS_REENTRANT (MICROPY_FATFS_REENTRANT)
 #else
-#define _FS_REENTRANT   0
+#define FF_FS_REENTRANT 0
 #endif
 
 // milliseconds
 #ifdef MICROPY_FATFS_TIMEOUT
-#define _FS_TIMEOUT     (MICROPY_FATFS_TIMEOUT)
+#define FF_FS_TIMEOUT   (MICROPY_FATFS_TIMEOUT)
 #else
-#define _FS_TIMEOUT     1000
+#define FF_FS_TIMEOUT   1000
 #endif
 
 #ifdef MICROPY_FATFS_SYNC_T
-#define _SYNC_t         MICROPY_FATFS_SYNC_T
+#define FF_SYNC_t       MICROPY_FATFS_SYNC_T
 #else
-#define _SYNC_t         HANDLE
+#define FF_SYNC_t       HANDLE
 #endif
-/* The option _FS_REENTRANT switches the re-entrancy (thread safe) of the FatFs
+/* The option FF_FS_REENTRANT switches the re-entrancy (thread safe) of the FatFs
 /  module itself. Note that regardless of this option, file access to different
 /  volume is always re-entrant and volume control functions, f_mount(), f_mkfs()
 /  and f_fdisk() function, are always not re-entrant. Only file/directory access
 /  to the same volume is under control of this function.
 /
-/   0: Disable re-entrancy. _FS_TIMEOUT and _SYNC_t have no effect.
+/   0: Disable re-entrancy. FF_FS_TIMEOUT and FF_SYNC_t have no effect.
 /   1: Enable re-entrancy. Also user provided synchronization handlers,
 /      ff_req_grant(), ff_rel_grant(), ff_del_syncobj() and ff_cre_syncobj()
 /      function, must be added to the project. Samples are available in
 /      option/syscall.c.
 /
-/  The _FS_TIMEOUT defines timeout period in unit of time tick.
-/  The _SYNC_t defines O/S dependent sync object type. e.g. HANDLE, ID, OS_EVENT*,
-/  SemaphoreHandle_t and etc.. A header file for O/S definitions needs to be
+/  The FF_FS_TIMEOUT defines timeout period in unit of time tick.
+/  The FF_SYNC_t defines O/S dependent sync object type. e.g. HANDLE, ID, OS_EVENT*,
+/  SemaphoreHandle_t and etc. A header file for O/S definitions needs to be
 /  included somewhere in the scope of ff.h. */
 
-/* #include <windows.h> // O/S definitions  */
 
 
 /*--- End of configuration options ---*/

lib/oofatfs/ffunicode.c

@@ -1,33 +1,46 @@
 /*------------------------------------------------------------------------*/
-/* Unicode - Local code bidirectional converter  (C)ChaN, 2015            */
-/* (SBCS code pages)                                                      */
+/* Unicode handling functions for FatFs R0.13c                            */
 /*------------------------------------------------------------------------*/
-/*  437   U.S.
-/   720   Arabic
-/   737   Greek
-/   771   KBL
-/   775   Baltic
-/   850   Latin 1
-/   852   Latin 2
-/   855   Cyrillic
-/   857   Turkish
-/   860   Portuguese
-/   861   Icelandic
-/   862   Hebrew
-/   863   Canadian French
-/   864   Arabic
-/   865   Nordic
-/   866   Russian
-/   869   Greek 2
+/* This module will occupy a huge memory in the .const section when the    /
+/  FatFs is configured for LFN with DBCS. If the system has any Unicode    /
+/  utilitiy for the code conversion, this module should be modified to use /
+/  that function to avoid silly memory consumption.                        /
+/-------------------------------------------------------------------------*/
+/*
+/ Copyright (C) 2018, ChaN, all right reserved.
+/
+/ FatFs module is an open source software. Redistribution and use of FatFs in
+/ source and binary forms, with or without modification, are permitted provided
+/ that the following condition is met:
+/
+/ 1. Redistributions of source code must retain the above copyright notice,
+/    this condition and the following disclaimer.
+/
+/ This software is provided by the copyright holder and contributors "AS IS"
+/ and any warranties related to this software are DISCLAIMED.
+/ The copyright owner or contributors be NOT LIABLE for any damages caused
+/ by use of this software.
 */
 
-#include "../ff.h"
+
+#include "ff.h"
+
+#if FF_USE_LFN  /* This module will be blanked at non-LFN configuration */
+
+#if FF_DEFINED != 86604 /* Revision ID */
+#error Wrong include file (ff.h).
+#endif
+
+#define MERGE2(a, b) a ## b
+#define CVTBL(tbl, cp) MERGE2(tbl, cp)
 
 
-#if _CODE_PAGE == 437
-#define _TBLDEF 1
-static
-const WCHAR Tbl[] = {   /*  CP437(0x80-0xFF) to Unicode conversion table */
+/*------------------------------------------------------------------------*/
+/* Code Conversion Tables                                                 */
+/*------------------------------------------------------------------------*/
+
+#if FF_CODE_PAGE == 437 || FF_CODE_PAGE == 0
+static const WCHAR uc437[] = {  /*  CP437(U.S.) to Unicode conversion table */
     0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x00E0, 0x00E5, 0x00E7, 0x00EA, 0x00EB, 0x00E8, 0x00EF, 0x00EE, 0x00EC, 0x00C4, 0x00C5,
     0x00C9, 0x00E6, 0x00C6, 0x00F4, 0x00F6, 0x00F2, 0x00FB, 0x00F9, 0x00FF, 0x00D6, 0x00DC, 0x00A2, 0x00A3, 0x00A5, 0x20A7, 0x0192,
     0x00E1, 0x00ED, 0x00F3, 0x00FA, 0x00F1, 0x00D1, 0x00AA, 0x00BA, 0x00BF, 0x2310, 0x00AC, 0x00BD, 0x00BC, 0x00A1, 0x00AB, 0x00BB,
@@ -37,11 +50,9 @@ const WCHAR Tbl[] = {   /*  CP437(0x80-0xFF) to Unicode conversion table */
     0x03B1, 0x00DF, 0x0393, 0x03C0, 0x03A3, 0x03C3, 0x00B5, 0x03C4, 0x03A6, 0x0398, 0x03A9, 0x03B4, 0x221E, 0x03C6, 0x03B5, 0x2229,
     0x2261, 0x00B1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00F7, 0x2248, 0x00B0, 0x2219, 0x00B7, 0x221A, 0x207F, 0x00B2, 0x25A0, 0x00A0
 };
-
-#elif _CODE_PAGE == 720
-#define _TBLDEF 1
-static
-const WCHAR Tbl[] = {   /*  CP720(0x80-0xFF) to Unicode conversion table */
+#endif
+#if FF_CODE_PAGE == 720 || FF_CODE_PAGE == 0
+static const WCHAR uc720[] = {  /*  CP720(Arabic) to Unicode conversion table */
     0x0000, 0x0000, 0x00E9, 0x00E2, 0x0000, 0x00E0, 0x0000, 0x00E7, 0x00EA, 0x00EB, 0x00E8, 0x00EF, 0x00EE, 0x0000, 0x0000, 0x0000,
     0x0000, 0x0651, 0x0652, 0x00F4, 0x00A4, 0x0640, 0x00FB, 0x00F9, 0x0621, 0x0622, 0x0623, 0x0624, 0x00A3, 0x0625, 0x0626, 0x0627,
     0x0628, 0x0629, 0x062A, 0x062B, 0x062C, 0x062D, 0x062E, 0x062F, 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x00AB, 0x00BB,
@@ -51,11 +62,9 @@ const WCHAR Tbl[] = {   /*  CP720(0x80-0xFF) to Unicode conversion table */
     0x0636, 0x0637, 0x0638, 0x0639, 0x063A, 0x0641, 0x00B5, 0x0642, 0x0643, 0x0644, 0x0645, 0x0646, 0x0647, 0x0648, 0x0649, 0x064A,
     0x2261, 0x064B, 0x064C, 0x064D, 0x064E, 0x064F, 0x0650, 0x2248, 0x00B0, 0x2219, 0x00B7, 0x221A, 0x207F, 0x00B2, 0x25A0, 0x00A0
 };
-
-#elif _CODE_PAGE == 737
-#define _TBLDEF 1
-static
-const WCHAR Tbl[] = {   /*  CP737(0x80-0xFF) to Unicode conversion table */
+#endif
+#if FF_CODE_PAGE == 737 || FF_CODE_PAGE == 0
+static const WCHAR uc737[] = {  /*  CP737(Greek) to Unicode conversion table */
     0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, 0x0398, 0x0399, 0x039A, 0x039B, 0x039C, 0x039D, 0x039E, 0x039F, 0x03A0,
     0x03A1, 0x03A3, 0x03A4, 0x03A5, 0x03A6, 0x03A7, 0x03A8, 0x03A9, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7, 0x03B8,
     0x03B9, 0x03BA, 0x03BB, 0x03BC, 0x03BD, 0x03BE, 0x03BF, 0x03C0, 0x03C1, 0x03C3, 0x03C2, 0x03C4, 0x03C5, 0x03C6, 0x03C7, 0x03C8,
@@ -65,11 +74,9 @@ const WCHAR Tbl[] = {   /*  CP737(0x80-0xFF) to Unicode conversion table */
     0x03C9, 0x03AC, 0x03AD, 0x03AE, 0x03CA, 0x03AF, 0x03CC, 0x03CD, 0x03CB, 0x03CE, 0x0386, 0x0388, 0x0389, 0x038A, 0x038C, 0x038E,
     0x038F, 0x00B1, 0x2265, 0x2264, 0x03AA, 0x03AB, 0x00F7, 0x2248, 0x00B0, 0x2219, 0x00B7, 0x221A, 0x207F, 0x00B2, 0x25A0, 0x00A0
 };
-
-#elif _CODE_PAGE == 771
-#define _TBLDEF 1
-static
-const WCHAR Tbl[] = {   /*  CP771(0x80-0xFF) to Unicode conversion table */
+#endif
+#if FF_CODE_PAGE == 771 || FF_CODE_PAGE == 0
+static const WCHAR uc771[] = {  /*  CP771(KBL) to Unicode conversion table */
     0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F,
     0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F,
     0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F,
@@ -79,11 +86,9 @@ const WCHAR Tbl[] = {   /*  CP771(0x80-0xFF) to Unicode conversion table */
     0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F,
     0x0118, 0x0119, 0x0116, 0x0117, 0x012E, 0x012F, 0x0160, 0x0161, 0x0172, 0x0173, 0x016A, 0x016B, 0x017D, 0x017E, 0x25A0, 0x00A0
 };
-
-#elif _CODE_PAGE == 775
-#define _TBLDEF 1
-static
-const WCHAR Tbl[] = {   /*  CP775(0x80-0xFF) to Unicode conversion table */
+#endif
+#if FF_CODE_PAGE == 775 || FF_CODE_PAGE == 0
+static const WCHAR uc775[] = {  /*  CP775(Baltic) to Unicode conversion table */
     0x0106, 0x00FC, 0x00E9, 0x0101, 0x00E4, 0x0123, 0x00E5, 0x0107, 0x0142, 0x0113, 0x0156, 0x0157, 0x012B, 0x0179, 0x00C4, 0x00C5,
     0x00C9, 0x00E6, 0x00C6, 0x014D, 0x00F6, 0x0122, 0x00A2, 0x015A, 0x015B, 0x00D6, 0x00DC, 0x00F8, 0x00A3, 0x00D8, 0x00D7, 0x00A4,
     0x0100, 0x012A, 0x00F3, 0x017B, 0x017C, 0x017A, 0x201D, 0x00A6, 0x00A9, 0x00AE, 0x00AC, 0x00BD, 0x00BC, 0x0141, 0x00AB, 0x00BB,
@@ -93,11 +98,9 @@ const WCHAR Tbl[] = {   /*  CP775(0x80-0xFF) to Unicode conversion table */
     0x00D3, 0x00DF, 0x014C, 0x0143, 0x00F5, 0x00D5, 0x00B5, 0x0144, 0x0136, 0x0137, 0x013B, 0x013C, 0x0146, 0x0112, 0x0145, 0x2019,
     0x00AD, 0x00B1, 0x201C, 0x00BE, 0x00B6, 0x00A7, 0x00F7, 0x201E, 0x00B0, 0x2219, 0x00B7, 0x00B9, 0x00B3, 0x00B2, 0x25A0, 0x00A0
 };
-
-#elif _CODE_PAGE == 850
-#define _TBLDEF 1
-static
-const WCHAR Tbl[] = {   /*  CP850(0x80-0xFF) to Unicode conversion table */
+#endif
+#if FF_CODE_PAGE == 850 || FF_CODE_PAGE == 0
+static const WCHAR uc850[] = {  /*  CP850(Latin 1) to Unicode conversion table */
     0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x00E0, 0x00E5, 0x00E7, 0x00EA, 0x00EB, 0x00E8, 0x00EF, 0x00EE, 0x00EC, 0x00C4, 0x00C5,
     0x00C9, 0x00E6, 0x00C6, 0x00F4, 0x00F6, 0x00F2, 0x00FB, 0x00F9, 0x00FF, 0x00D6, 0x00DC, 0x00F8, 0x00A3, 0x00D8, 0x00D7, 0x0192,
     0x00E1, 0x00ED, 0x00F3, 0x00FA, 0x00F1, 0x00D1, 0x00AA, 0x00BA, 0x00BF, 0x00AE, 0x00AC, 0x00BD, 0x00BC, 0x00A1, 0x00AB, 0x00BB,
@@ -107,11 +110,9 @@ const WCHAR Tbl[] = {   /*  CP850(0x80-0xFF) to Unicode conversion table */
     0x00D3, 0x00DF, 0x00D4, 0x00D2, 0x00F5, 0x00D5, 0x00B5, 0x00FE, 0x00DE, 0x00DA, 0x00DB, 0x00D9, 0x00FD, 0x00DD, 0x00AF, 0x00B4,
     0x00AD, 0x00B1, 0x2017, 0x00BE, 0x00B6, 0x00A7, 0x00F7, 0x00B8, 0x00B0, 0x00A8, 0x00B7, 0x00B9, 0x00B3, 0x00B2, 0x25A0, 0x00A0
 };
-
-#elif _CODE_PAGE == 852
-#define _TBLDEF 1
-static
-const WCHAR Tbl[] = {   /*  CP852(0x80-0xFF) to Unicode conversion table */
+#endif
+#if FF_CODE_PAGE == 852 || FF_CODE_PAGE == 0
+static const WCHAR uc852[] = {  /*  CP852(Latin 2) to Unicode conversion table */
     0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x016F, 0x0107, 0x00E7, 0x0142, 0x00EB, 0x0150, 0x0151, 0x00EE, 0x0179, 0x00C4, 0x0106,
     0x00C9, 0x0139, 0x013A, 0x00F4, 0x00F6, 0x013D, 0x013E, 0x015A, 0x015B, 0x00D6, 0x00DC, 0x0164, 0x0165, 0x0141, 0x00D7, 0x010D,
     0x00E1, 0x00ED, 0x00F3, 0x00FA, 0x0104, 0x0105, 0x017D, 0x017E, 0x0118, 0x0119, 0x00AC, 0x017A, 0x010C, 0x015F, 0x00AB, 0x00BB,
@@ -121,11 +122,9 @@ const WCHAR Tbl[] = {   /*  CP852(0x80-0xFF) to Unicode conversion table */
     0x00D3, 0x00DF, 0x00D4, 0x0143, 0x0144, 0x0148, 0x0160, 0x0161, 0x0154, 0x00DA, 0x0155, 0x0170, 0x00FD, 0x00DD, 0x0163, 0x00B4,
     0x00AD, 0x02DD, 0x02DB, 0x02C7, 0x02D8, 0x00A7, 0x00F7, 0x00B8, 0x00B0, 0x00A8, 0x02D9, 0x0171, 0x0158, 0x0159, 0x25A0, 0x00A0
 };
-
-#elif _CODE_PAGE == 855
-#define _TBLDEF 1
-static
-const WCHAR Tbl[] = {   /*  CP855(0x80-0xFF) to Unicode conversion table */
+#endif
+#if FF_CODE_PAGE == 855 || FF_CODE_PAGE == 0
+static const WCHAR uc855[] = {  /*  CP855(Cyrillic) to Unicode conversion table */
     0x0452, 0x0402, 0x0453, 0x0403, 0x0451, 0x0401, 0x0454, 0x0404, 0x0455, 0x0405, 0x0456, 0x0406, 0x0457, 0x0407, 0x0458, 0x0408,
     0x0459, 0x0409, 0x045A, 0x040A, 0x045B, 0x040B, 0x045C, 0x040C, 0x045E, 0x040E, 0x045F, 0x040F, 0x044E, 0x042E, 0x044A, 0x042A,
     0x0430, 0x0410, 0x0431, 0x0411, 0x0446, 0x0426, 0x0434, 0x0414, 0x0435, 0x0415, 0x0444, 0x0424, 0x0433, 0x0413, 0x00AB, 0x00BB,
@@ -135,11 +134,9 @@ const WCHAR Tbl[] = {   /*  CP855(0x80-0xFF) to Unicode conversion table */
     0x042F, 0x0440, 0x0420, 0x0441, 0x0421, 0x0442, 0x0422, 0x0443, 0x0423, 0x0436, 0x0416, 0x0432, 0x0412, 0x044C, 0x042C, 0x2116,
     0x00AD, 0x044B, 0x042B, 0x0437, 0x0417, 0x0448, 0x0428, 0x044D, 0x042D, 0x0449, 0x0429, 0x0447, 0x0427, 0x00A7, 0x25A0, 0x00A0
 };
-
-#elif _CODE_PAGE == 857
-#define _TBLDEF 1
-static
-const WCHAR Tbl[] = {   /*  CP857(0x80-0xFF) to Unicode conversion table */
+#endif
+#if FF_CODE_PAGE == 857 || FF_CODE_PAGE == 0
+static const WCHAR uc857[] = {  /*  CP857(Turkish) to Unicode conversion table */
     0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x00E0, 0x00E5, 0x00E7, 0x00EA, 0x00EB, 0x00E8, 0x00EF, 0x00EE, 0x0131, 0x00C4, 0x00C5,
     0x00C9, 0x00E6, 0x00C6, 0x00F4, 0x00F6, 0x00F2, 0x00FB, 0x00F9, 0x0130, 0x00D6, 0x00DC, 0x00F8, 0x00A3, 0x00D8, 0x015E, 0x015F,
     0x00E1, 0x00ED, 0x00F3, 0x00FA, 0x00F1, 0x00D1, 0x011E, 0x011F, 0x00BF, 0x00AE, 0x00AC, 0x00BD, 0x00BC, 0x00A1, 0x00AB, 0x00BB,
@@ -149,11 +146,9 @@ const WCHAR Tbl[] = {   /*  CP857(0x80-0xFF) to Unicode conversion table */
     0x00D3, 0x00DF, 0x00D4, 0x00D2, 0x00F5, 0x00D5, 0x00B5, 0x0000, 0x00D7, 0x00DA, 0x00DB, 0x00D9, 0x00EC, 0x00FF, 0x00AF, 0x00B4,
     0x00AD, 0x00B1, 0x0000, 0x00BE, 0x00B6, 0x00A7, 0x00F7, 0x00B8, 0x00B0, 0x00A8, 0x00B7, 0x00B9, 0x00B3, 0x00B2, 0x25A0, 0x00A0
 };
-
-#elif _CODE_PAGE == 860
-#define _TBLDEF 1
-static
-const WCHAR Tbl[] = {   /*  CP860(0x80-0xFF) to Unicode conversion table */
+#endif
+#if FF_CODE_PAGE == 860 || FF_CODE_PAGE == 0
+static const WCHAR uc860[] = {  /*  CP860(Portuguese) to Unicode conversion table */
     0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E3, 0x00E0, 0x00C1, 0x00E7, 0x00EA, 0x00CA, 0x00E8, 0x00CD, 0x00D4, 0x00EC, 0x00C3, 0x00C2,
     0x00C9, 0x00C0, 0x00C8, 0x00F4, 0x00F5, 0x00F2, 0x00DA, 0x00F9, 0x00CC, 0x00D5, 0x00DC, 0x00A2, 0x00A3, 0x00D9, 0x20A7, 0x00D3,
     0x00E1, 0x00ED, 0x00F3, 0x00FA, 0x00F1, 0x00D1, 0x00AA, 0x00BA, 0x00BF, 0x00D2, 0x00AC, 0x00BD, 0x00BC, 0x00A1, 0x00AB, 0x00BB,
@@ -163,11 +158,9 @@ const WCHAR Tbl[] = {   /*  CP860(0x80-0xFF) to Unicode conversion table */
     0x03B1, 0x00DF, 0x0393, 0x03C0, 0x03A3, 0x03C3, 0x00B5, 0x03C4, 0x03A6, 0x0398, 0x03A9, 0x03B4, 0x221E, 0x03C6, 0x03B5, 0x2229,
     0x2261, 0x00B1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00F7, 0x2248, 0x00B0, 0x2219, 0x00B7, 0x221A, 0x207F, 0x00B2, 0x25A0, 0x00A0
 };
-
-#elif _CODE_PAGE == 861
-#define _TBLDEF 1
-static
-const WCHAR Tbl[] = {   /*  CP861(0x80-0xFF) to Unicode conversion table */
+#endif
+#if FF_CODE_PAGE == 861 || FF_CODE_PAGE == 0
+static const WCHAR uc861[] = {  /*  CP861(Icelandic) to Unicode conversion table */
     0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x00E0, 0x00E6, 0x00E7, 0x00EA, 0x00EB, 0x00E8, 0x00D0, 0x00F0, 0x00DE, 0x00C4, 0x00C5,
     0x00C9, 0x00E6, 0x00C6, 0x00F4, 0x00F6, 0x00FE, 0x00FB, 0x00DD, 0x00FD, 0x00D6, 0x00DC, 0x00F8, 0x00A3, 0x00D8, 0x20A7, 0x0192,
     0x00E1, 0x00ED, 0x00F3, 0x00FA, 0x00C1, 0x00CD, 0x00D3, 0x00DA, 0x00BF, 0x2310, 0x00AC, 0x00BD, 0x00BC, 0x00A1, 0x00AB, 0x00BB,
@@ -177,11 +170,9 @@ const WCHAR Tbl[] = {   /*  CP861(0x80-0xFF) to Unicode conversion table */
     0x03B1, 0x00DF, 0x0393, 0x03C0, 0x03A3, 0x03C3, 0x00B5, 0x03C4, 0x03A6, 0x0398, 0x03A9, 0x03B4, 0x221E, 0x03C6, 0x03B5, 0x2229,
     0x2261, 0x00B1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00F7, 0x2248, 0x00B0, 0x2219, 0x00B7, 0x221A, 0x207F, 0x00B2, 0x25A0, 0x00A0
 };
-
-#elif _CODE_PAGE == 862
-#define _TBLDEF 1
-static
-const WCHAR Tbl[] = {   /*  CP862(0x80-0xFF) to Unicode conversion table */
+#endif
+#if FF_CODE_PAGE == 862 || FF_CODE_PAGE == 0
+static const WCHAR uc862[] = {  /*  CP862(Hebrew) to Unicode conversion table */
     0x05D0, 0x05D1, 0x05D2, 0x05D3, 0x05D4, 0x05D5, 0x05D6, 0x05D7, 0x05D8, 0x05D9, 0x05DA, 0x05DB, 0x05DC, 0x05DD, 0x05DE, 0x05DF,
     0x05E0, 0x05E1, 0x05E2, 0x05E3, 0x05E4, 0x05E5, 0x05E6, 0x05E7, 0x05E8, 0x05E9, 0x05EA, 0x00A2, 0x00A3, 0x00A5, 0x20A7, 0x0192,
     0x00E1, 0x00ED, 0x00F3, 0x00FA, 0x00F1, 0x00D1, 0x00AA, 0x00BA, 0x00BF, 0x2310, 0x00AC, 0x00BD, 0x00BC, 0x00A1, 0x00AB, 0x00BB,
@@ -191,11 +182,9 @@ const WCHAR Tbl[] = {   /*  CP862(0x80-0xFF) to Unicode conversion table */
     0x03B1, 0x00DF, 0x0393, 0x03C0, 0x03A3, 0x03C3, 0x00B5, 0x03C4, 0x03A6, 0x0398, 0x03A9, 0x03B4, 0x221E, 0x03C6, 0x03B5, 0x2229,
     0x2261, 0x00B1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00F7, 0x2248, 0x00B0, 0x2219, 0x00B7, 0x221A, 0x207F, 0x00B2, 0x25A0, 0x00A0
 };
-
-#elif _CODE_PAGE == 863
-#define _TBLDEF 1
-static
-const WCHAR Tbl[] = {   /*  CP863(0x80-0xFF) to Unicode conversion table */
+#endif
+#if FF_CODE_PAGE == 863 || FF_CODE_PAGE == 0
+static const WCHAR uc863[] = {  /*  CP863(Canadian French) to Unicode conversion table */
     0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00C2, 0x00E0, 0x00B6, 0x00E7, 0x00EA, 0x00EB, 0x00E8, 0x00EF, 0x00EE, 0x00EC, 0x2017, 0x00C0,
     0x00C9, 0x00C8, 0x00CA, 0x00F4, 0x00CB, 0x00CF, 0x00FB, 0x00F9, 0x00A4, 0x00D4, 0x00DC, 0x00A2, 0x00A3, 0x00D9, 0x00DB, 0x0192,
     0x00A6, 0x00B4, 0x00F3, 0x00FA, 0x00A8, 0x00BB, 0x00B3, 0x00AF, 0x00CE, 0x3210, 0x00AC, 0x00BD, 0x00BC, 0x00BE, 0x00AB, 0x00BB,
@@ -205,11 +194,9 @@ const WCHAR Tbl[] = {   /*  CP863(0x80-0xFF) to Unicode conversion table */
     0x03B1, 0x00DF, 0x0393, 0x03C0, 0x03A3, 0x03C3, 0x00B5, 0x03C4, 0x03A6, 0x0398, 0x03A9, 0x03B4, 0x221E, 0x03C6, 0x03B5, 0x2219,
     0x2261, 0x00B1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00F7, 0x2248, 0x00B0, 0x2219, 0x00B7, 0x221A, 0x207F, 0x00B2, 0x25A0, 0x00A0
 };
-
-#elif _CODE_PAGE == 864
-#define _TBLDEF 1
-static
-const WCHAR Tbl[] = {   /*  CP864(0x80-0xFF) to Unicode conversion table */
+#endif
+#if FF_CODE_PAGE == 864 || FF_CODE_PAGE == 0
+static const WCHAR uc864[] = {  /*  CP864(Arabic) to Unicode conversion table */
     0x00B0, 0x00B7, 0x2219, 0x221A, 0x2592, 0x2500, 0x2502, 0x253C, 0x2524, 0x252C, 0x251C, 0x2534, 0x2510, 0x250C, 0x2514, 0x2518,
     0x03B2, 0x221E, 0x03C6, 0x00B1, 0x00BD, 0x00BC, 0x2248, 0x00AB, 0x00BB, 0xFEF7, 0xFEF8, 0x0000, 0x0000, 0xFEFB, 0xFEFC, 0x0000,
     0x00A0, 0x00AD, 0xFE82, 0x00A3, 0x00A4, 0xFE84, 0x0000, 0x20AC, 0xFE8E, 0xFE8F, 0xFE95, 0xFE99, 0x060C, 0xFE9D, 0xFEA1, 0xFEA5,
@@ -219,11 +206,9 @@ const WCHAR Tbl[] = {   /*  CP864(0x80-0xFF) to Unicode conversion table */
     0x0640, 0xFED3, 0xFED7, 0xFEDB, 0xFEDF, 0xFEE3, 0xFEE7, 0xFEEB, 0xFEED, 0xFEEF, 0xFEF3, 0xFEBD, 0xFECC, 0xFECE, 0xFECD, 0xFEE1,
     0xFE7D, 0x0651, 0xFEE5, 0xFEE9, 0xFEEC, 0xFEF0, 0xFEF2, 0xFED0, 0xFED5, 0xFEF5, 0xFEF6, 0xFEDD, 0xFED9, 0xFEF1, 0x25A0, 0x0000
 };
-
-#elif _CODE_PAGE == 865
-#define _TBLDEF 1
-static
-const WCHAR Tbl[] = {   /*  CP865(0x80-0xFF) to Unicode conversion table */
+#endif
+#if FF_CODE_PAGE == 865 || FF_CODE_PAGE == 0
+static const WCHAR uc865[] = {  /*  CP865(Nordic) to Unicode conversion table */
     0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x00E0, 0x00E5, 0x00E7, 0x00EA, 0x00EB, 0x00E8, 0x00EF, 0x00EE, 0x00EC, 0x00C4, 0x00C5,
     0x00C5, 0x00E6, 0x00C6, 0x00F4, 0x00F6, 0x00F2, 0x00FB, 0x00F9, 0x00FF, 0x00D6, 0x00DC, 0x00F8, 0x00A3, 0x00D8, 0x20A7, 0x0192,
     0x00E1, 0x00ED, 0x00F3, 0x00FA, 0x00F1, 0x00D1, 0x00AA, 0x00BA, 0x00BF, 0x2310, 0x00AC, 0x00BD, 0x00BC, 0x00A1, 0x00AB, 0x00A4,
@@ -233,11 +218,9 @@ const WCHAR Tbl[] = {   /*  CP865(0x80-0xFF) to Unicode conversion table */
     0x03B1, 0x00DF, 0x0393, 0x03C0, 0x03A3, 0x03C3, 0x00B5, 0x03C4, 0x03A6, 0x0398, 0x03A9, 0x03B4, 0x221E, 0x03C6, 0x03B5, 0x2229,
     0x2261, 0x00B1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00F7, 0x2248, 0x00B0, 0x2219, 0x00B7, 0x221A, 0x207F, 0x00B2, 0x25A0, 0x00A0
 };
-
-#elif _CODE_PAGE == 866
-#define _TBLDEF 1
-static
-const WCHAR Tbl[] = {   /*  CP866(0x80-0xFF) to Unicode conversion table */
+#endif
+#if FF_CODE_PAGE == 866 || FF_CODE_PAGE == 0
+static const WCHAR uc866[] = {  /*  CP866(Russian) to Unicode conversion table */
     0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F,
     0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F,
     0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F,
@@ -247,11 +230,9 @@ const WCHAR Tbl[] = {   /*  CP866(0x80-0xFF) to Unicode conversion table */
     0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F,
     0x0401, 0x0451, 0x0404, 0x0454, 0x0407, 0x0457, 0x040E, 0x045E, 0x00B0, 0x2219, 0x00B7, 0x221A, 0x2116, 0x00A4, 0x25A0, 0x00A0
 };
-
-#elif _CODE_PAGE == 869
-#define _TBLDEF 1
-static
-const WCHAR Tbl[] = {   /*  CP869(0x80-0xFF) to Unicode conversion table */
+#endif
+#if FF_CODE_PAGE == 869 || FF_CODE_PAGE == 0
+static const WCHAR uc869[] = {  /*  CP869(Greek 2) to Unicode conversion table */
     0x00B7, 0x00B7, 0x00B7, 0x00B7, 0x00B7, 0x00B7, 0x0386, 0x00B7, 0x00B7, 0x00AC, 0x00A6, 0x2018, 0x2019, 0x0388, 0x2015, 0x0389,
     0x038A, 0x03AA, 0x038C, 0x00B7, 0x00B7, 0x038E, 0x03AB, 0x00A9, 0x038F, 0x00B2, 0x00B3, 0x03AC, 0x00A3, 0x03AD, 0x03AE, 0x03AF,
     0x03CA, 0x0390, 0x03CC, 0x03CD, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, 0x00BD, 0x0398, 0x0399, 0x00AB, 0x00BB,
@@ -261,36 +242,32 @@ const WCHAR Tbl[] = {   /*  CP869(0x80-0xFF) to Unicode conversion table */
     0x03B6, 0x03B7, 0x03B8, 0x03B9, 0x03BA, 0x03BB, 0x03BC, 0x03BD, 0x03BE, 0x03BF, 0x03C0, 0x03C1, 0x03C3, 0x03C2, 0x03C4, 0x0384,
     0x00AD, 0x00B1, 0x03C5, 0x03C6, 0x03C7, 0x00A7, 0x03C8, 0x0385, 0x00B0, 0x00A8, 0x03C9, 0x03CB, 0x03B0, 0x03CE, 0x25A0, 0x00A0
 };
-
-#endif
-
-
-#if !_TBLDEF || !_USE_LFN
-#error This file is not needed at current configuration. Remove from the project.
 #endif
 
 
 
 
-WCHAR ff_convert (  /* Converted character, Returns zero on error */
-    WCHAR   chr,    /* Character code to be converted */
-    UINT    dir     /* 0: Unicode to OEM code, 1: OEM code to Unicode */
+/*------------------------------------------------------------------------*/
+/* OEM <==> Unicode conversions for static code page configuration        */
+/* SBCS fixed code page                                                   */
+/*------------------------------------------------------------------------*/
+
+#if FF_CODE_PAGE != 0 && FF_CODE_PAGE < 900
+WCHAR ff_uni2oem (  /* Returns OEM code character, zero on error */
+    DWORD   uni,    /* UTF-16 encoded character to be converted */
+    WORD    cp      /* Code page for the conversion */
 )
 {
-    WCHAR c;
+    WCHAR c = 0;
+    const WCHAR *p = CVTBL(uc, FF_CODE_PAGE);
 
 
-    if (chr < 0x80) {   /* ASCII */
-        c = chr;
+    if (uni < 0x80) {   /* ASCII? */
+        c = (WCHAR)uni;
 
-    } else {
-        if (dir) {      /* OEM code to Unicode */
-            c = (chr >= 0x100) ? 0 : Tbl[chr - 0x80];
-
-        } else {        /* Unicode to OEM code */
-            for (c = 0; c < 0x80; c++) {
-                if (chr == Tbl[c]) break;
-            }
+    } else {            /* Non-ASCII */
+        if (uni < 0x10000 && cp == FF_CODE_PAGE) {  /* Is it in BMP and valid code page? */
+            for (c = 0; c < 0x80 && uni != p[c]; c++) ;
             c = (c + 0x80) & 0xFF;
         }
     }
@@ -298,56 +275,314 @@ WCHAR ff_convert (  /* Converted character, Returns zero on error */
     return c;
 }
 
-
-
-WCHAR ff_wtoupper ( /* Returns upper converted character */
-    WCHAR chr       /* Unicode character to be upper converted (BMP only) */
+WCHAR ff_oem2uni (  /* Returns Unicode character, zero on error */
+    WCHAR   oem,    /* OEM code to be converted */
+    WORD    cp      /* Code page for the conversion */
 )
 {
-    /* Compressed upper conversion table */
-    static const WCHAR cvt1[] = {   /* U+0000 - U+0FFF */
+    WCHAR c = 0;
+    const WCHAR *p = CVTBL(uc, FF_CODE_PAGE);
+
+
+    if (oem < 0x80) {   /* ASCII? */
+        c = oem;
+
+    } else {            /* Extended char */
+        if (cp == FF_CODE_PAGE) {   /* Is it a valid code page? */
+            if (oem < 0x100) c = p[oem - 0x80];
+        }
+    }
+
+    return c;
+}
+
+#endif
+
+
+
+/*------------------------------------------------------------------------*/
+/* OEM <==> Unicode conversions for static code page configuration        */
+/* DBCS fixed code page                                                   */
+/*------------------------------------------------------------------------*/
+
+#if FF_CODE_PAGE >= 900
+WCHAR ff_uni2oem (  /* Returns OEM code character, zero on error */
+    DWORD   uni,    /* UTF-16 encoded character to be converted */
+    WORD    cp      /* Code page for the conversion */
+)
+{
+    const WCHAR *p;
+    WCHAR c = 0, uc;
+    UINT i = 0, n, li, hi;
+
+
+    if (uni < 0x80) {   /* ASCII? */
+        c = (WCHAR)uni;
+
+    } else {            /* Non-ASCII */
+        if (uni < 0x10000 && cp == FF_CODE_PAGE) {  /* Is it in BMP and valid code page? */
+            uc = (WCHAR)uni;
+            p = CVTBL(uni2oem, FF_CODE_PAGE);
+            hi = sizeof CVTBL(uni2oem, FF_CODE_PAGE) / 4 - 1;
+            li = 0;
+            for (n = 16; n; n--) {
+                i = li + (hi - li) / 2;
+                if (uc == p[i * 2]) break;
+                if (uc > p[i * 2]) {
+                    li = i;
+                } else {
+                    hi = i;
+                }
+            }
+            if (n != 0) c = p[i * 2 + 1];
+        }
+    }
+
+    return c;
+}
+
+
+WCHAR ff_oem2uni (  /* Returns Unicode character, zero on error */
+    WCHAR   oem,    /* OEM code to be converted */
+    WORD    cp      /* Code page for the conversion */
+)
+{
+    const WCHAR *p;
+    WCHAR c = 0;
+    UINT i = 0, n, li, hi;
+
+
+    if (oem < 0x80) {   /* ASCII? */
+        c = oem;
+
+    } else {            /* Extended char */
+        if (cp == FF_CODE_PAGE) {   /* Is it valid code page? */
+            p = CVTBL(oem2uni, FF_CODE_PAGE);
+            hi = sizeof CVTBL(oem2uni, FF_CODE_PAGE) / 4 - 1;
+            li = 0;
+            for (n = 16; n; n--) {
+                i = li + (hi - li) / 2;
+                if (oem == p[i * 2]) break;
+                if (oem > p[i * 2]) {
+                    li = i;
+                } else {
+                    hi = i;
+                }
+            }
+            if (n != 0) c = p[i * 2 + 1];
+        }
+    }
+
+    return c;
+}
+#endif
+
+
+
+/*------------------------------------------------------------------------*/
+/* OEM <==> Unicode conversions for dynamic code page configuration       */
+/*------------------------------------------------------------------------*/
+
+#if FF_CODE_PAGE == 0
+
+static const WORD cp_code[]          = {  437,   720,   737,   771,   775,   850,   852,   855,   857,   860,   861,   862,   863,   864,   865,   866,   869, 0};
+static const WCHAR* const cp_table[] = {uc437, uc720, uc737, uc771, uc775, uc850, uc852, uc855, uc857, uc860, uc861, uc862, uc863, uc864, uc865, uc866, uc869, 0};
+
+
+WCHAR ff_uni2oem (  /* Returns OEM code character, zero on error */
+    DWORD   uni,    /* UTF-16 encoded character to be converted */
+    WORD    cp      /* Code page for the conversion */
+)
+{
+    const WCHAR *p;
+    WCHAR c = 0, uc;
+    UINT i, n, li, hi;
+
+
+    if (uni < 0x80) {   /* ASCII? */
+        c = (WCHAR)uni;
+
+    } else {            /* Non-ASCII */
+        if (uni < 0x10000) { /* Is it in BMP? */
+            uc = (WCHAR)uni;
+            p = 0;
+            if (cp < 900) { /* SBCS */
+                for (i = 0; cp_code[i] != 0 && cp_code[i] != cp; i++) ;     /* Get conversion table */
+                p = cp_table[i];
+                if (p) {    /* Is it valid code page ? */
+                    for (c = 0; c < 0x80 && uc != p[c]; c++) ;  /* Find OEM code in the table */
+                    c = (c + 0x80) & 0xFF;
+                }
+            } else {    /* DBCS */
+                switch (cp) {   /* Get conversion table */
+                case 932 : p = uni2oem932; hi = sizeof uni2oem932 / 4 - 1; break;
+                case 936 : p = uni2oem936; hi = sizeof uni2oem936 / 4 - 1; break;
+                case 949 : p = uni2oem949; hi = sizeof uni2oem949 / 4 - 1; break;
+                case 950 : p = uni2oem950; hi = sizeof uni2oem950 / 4 - 1; break;
+                }
+                if (p) {    /* Is it valid code page? */
+                    li = 0;
+                    for (n = 16; n; n--) {  /* Find OEM code */
+                        i = li + (hi - li) / 2;
+                        if (uc == p[i * 2]) break;
+                        if (uc > p[i * 2]) {
+                            li = i;
+                        } else {
+                            hi = i;
+                        }
+                    }
+                    if (n != 0) c = p[i * 2 + 1];
+                }
+            }
+        }
+    }
+
+    return c;
+}
+
+
+WCHAR ff_oem2uni (  /* Returns Unicode character, zero on error */
+    WCHAR   oem,    /* OEM code to be converted (DBC if >=0x100) */
+    WORD    cp      /* Code page for the conversion */
+)
+{
+    const WCHAR *p;
+    WCHAR c = 0;
+    UINT i, n, li, hi;
+
+
+    if (oem < 0x80) {   /* ASCII? */
+        c = oem;
+
+    } else {            /* Extended char */
+        p = 0;
+        if (cp < 900) { /* SBCS */
+            for (i = 0; cp_code[i] != 0 && cp_code[i] != cp; i++) ;     /* Get table */
+            p = cp_table[i];
+            if (p) {    /* Is it a valid CP ? */
+                if (oem < 0x100) c = p[oem - 0x80];
+            }
+        } else {    /* DBCS */
+            switch (cp) {
+            case 932 : p = oem2uni932; hi = sizeof oem2uni932 / 4 - 1; break;
+            case 936 : p = oem2uni936; hi = sizeof oem2uni936 / 4 - 1; break;
+            case 949 : p = oem2uni949; hi = sizeof oem2uni949 / 4 - 1; break;
+            case 950 : p = oem2uni950; hi = sizeof oem2uni950 / 4 - 1; break;
+            }
+            if (p) {
+                li = 0;
+                for (n = 16; n; n--) {
+                    i = li + (hi - li) / 2;
+                    if (oem == p[i * 2]) break;
+                    if (oem > p[i * 2]) {
+                        li = i;
+                    } else {
+                        hi = i;
+                    }
+                }
+                if (n != 0) c = p[i * 2 + 1];
+            }
+        }
+    }
+
+    return c;
+}
+#endif
+
+
+
+/*------------------------------------------------------------------------*/
+/* Unicode up-case conversion                                             */
+/*------------------------------------------------------------------------*/
+
+DWORD ff_wtoupper ( /* Returns up-converted code point */
+    DWORD uni       /* Unicode code point to be up-converted */
+)
+{
+    const WORD *p;
+    WORD uc, bc, nc, cmd;
+    static const WORD cvt1[] = {    /* Compressed up conversion table for U+0000 - U+0FFF */
         /* Basic Latin */
         0x0061,0x031A,
         /* Latin-1 Supplement */
-        0x00E0,0x0317,  0x00F8,0x0307,  0x00FF,0x0001,0x0178,
+        0x00E0,0x0317,
+        0x00F8,0x0307,
+        0x00FF,0x0001,0x0178,
         /* Latin Extended-A */
-        0x0100,0x0130,  0x0132,0x0106,  0x0139,0x0110,  0x014A,0x012E,  0x0179,0x0106,
+        0x0100,0x0130,
+        0x0132,0x0106,
+        0x0139,0x0110,
+        0x014A,0x012E,
+        0x0179,0x0106,
         /* Latin Extended-B */
         0x0180,0x004D,0x0243,0x0181,0x0182,0x0182,0x0184,0x0184,0x0186,0x0187,0x0187,0x0189,0x018A,0x018B,0x018B,0x018D,0x018E,0x018F,0x0190,0x0191,0x0191,0x0193,0x0194,0x01F6,0x0196,0x0197,0x0198,0x0198,0x023D,0x019B,0x019C,0x019D,0x0220,0x019F,0x01A0,0x01A0,0x01A2,0x01A2,0x01A4,0x01A4,0x01A6,0x01A7,0x01A7,0x01A9,0x01AA,0x01AB,0x01AC,0x01AC,0x01AE,0x01AF,0x01AF,0x01B1,0x01B2,0x01B3,0x01B3,0x01B5,0x01B5,0x01B7,0x01B8,0x01B8,0x01BA,0x01BB,0x01BC,0x01BC,0x01BE,0x01F7,0x01C0,0x01C1,0x01C2,0x01C3,0x01C4,0x01C5,0x01C4,0x01C7,0x01C8,0x01C7,0x01CA,0x01CB,0x01CA,
-        0x01CD,0x0110,  0x01DD,0x0001,0x018E,  0x01DE,0x0112,  0x01F3,0x0003,0x01F1,0x01F4,0x01F4,  0x01F8,0x0128,
-        0x0222,0x0112,  0x023A,0x0009,0x2C65,0x023B,0x023B,0x023D,0x2C66,0x023F,0x0240,0x0241,0x0241,  0x0246,0x010A,
+        0x01CD,0x0110,
+        0x01DD,0x0001,0x018E,
+        0x01DE,0x0112,
+        0x01F3,0x0003,0x01F1,0x01F4,0x01F4,
+        0x01F8,0x0128,
+        0x0222,0x0112,
+        0x023A,0x0009,0x2C65,0x023B,0x023B,0x023D,0x2C66,0x023F,0x0240,0x0241,0x0241,
+        0x0246,0x010A,
         /* IPA Extensions */
         0x0253,0x0040,0x0181,0x0186,0x0255,0x0189,0x018A,0x0258,0x018F,0x025A,0x0190,0x025C,0x025D,0x025E,0x025F,0x0193,0x0261,0x0262,0x0194,0x0264,0x0265,0x0266,0x0267,0x0197,0x0196,0x026A,0x2C62,0x026C,0x026D,0x026E,0x019C,0x0270,0x0271,0x019D,0x0273,0x0274,0x019F,0x0276,0x0277,0x0278,0x0279,0x027A,0x027B,0x027C,0x2C64,0x027E,0x027F,0x01A6,0x0281,0x0282,0x01A9,0x0284,0x0285,0x0286,0x0287,0x01AE,0x0244,0x01B1,0x01B2,0x0245,0x028D,0x028E,0x028F,0x0290,0x0291,0x01B7,
         /* Greek, Coptic */
-        0x037B,0x0003,0x03FD,0x03FE,0x03FF,  0x03AC,0x0004,0x0386,0x0388,0x0389,0x038A,  0x03B1,0x0311,
-        0x03C2,0x0002,0x03A3,0x03A3,  0x03C4,0x0308,  0x03CC,0x0003,0x038C,0x038E,0x038F,  0x03D8,0x0118,
+        0x037B,0x0003,0x03FD,0x03FE,0x03FF,
+        0x03AC,0x0004,0x0386,0x0388,0x0389,0x038A,
+        0x03B1,0x0311,
+        0x03C2,0x0002,0x03A3,0x03A3,
+        0x03C4,0x0308,
+        0x03CC,0x0003,0x038C,0x038E,0x038F,
+        0x03D8,0x0118,
         0x03F2,0x000A,0x03F9,0x03F3,0x03F4,0x03F5,0x03F6,0x03F7,0x03F7,0x03F9,0x03FA,0x03FA,
         /* Cyrillic */
-        0x0430,0x0320,  0x0450,0x0710,  0x0460,0x0122,  0x048A,0x0136,  0x04C1,0x010E,  0x04CF,0x0001,0x04C0,  0x04D0,0x0144,
+        0x0430,0x0320,
+        0x0450,0x0710,
+        0x0460,0x0122,
+        0x048A,0x0136,
+        0x04C1,0x010E,
+        0x04CF,0x0001,0x04C0,
+        0x04D0,0x0144,
         /* Armenian */
         0x0561,0x0426,
 
-        0x0000
+        0x0000  /* EOT */
     };
-    static const WCHAR cvt2[] = {   /* U+1000 - U+FFFF */
+    static const WORD cvt2[] = {    /* Compressed up conversion table for U+1000 - U+FFFF */
         /* Phonetic Extensions */
         0x1D7D,0x0001,0x2C63,
         /* Latin Extended Additional */
-        0x1E00,0x0196,  0x1EA0,0x015A,
+        0x1E00,0x0196,
+        0x1EA0,0x015A,
         /* Greek Extended */
-        0x1F00,0x0608,  0x1F10,0x0606,  0x1F20,0x0608,  0x1F30,0x0608,  0x1F40,0x0606,
-        0x1F51,0x0007,0x1F59,0x1F52,0x1F5B,0x1F54,0x1F5D,0x1F56,0x1F5F,  0x1F60,0x0608,
+        0x1F00,0x0608,
+        0x1F10,0x0606,
+        0x1F20,0x0608,
+        0x1F30,0x0608,
+        0x1F40,0x0606,
+        0x1F51,0x0007,0x1F59,0x1F52,0x1F5B,0x1F54,0x1F5D,0x1F56,0x1F5F,
+        0x1F60,0x0608,
         0x1F70,0x000E,0x1FBA,0x1FBB,0x1FC8,0x1FC9,0x1FCA,0x1FCB,0x1FDA,0x1FDB,0x1FF8,0x1FF9,0x1FEA,0x1FEB,0x1FFA,0x1FFB,
-        0x1F80,0x0608,  0x1F90,0x0608,  0x1FA0,0x0608,  0x1FB0,0x0004,0x1FB8,0x1FB9,0x1FB2,0x1FBC,
-        0x1FCC,0x0001,0x1FC3,  0x1FD0,0x0602,  0x1FE0,0x0602,  0x1FE5,0x0001,0x1FEC,  0x1FF2,0x0001,0x1FFC,
+        0x1F80,0x0608,
+        0x1F90,0x0608,
+        0x1FA0,0x0608,
+        0x1FB0,0x0004,0x1FB8,0x1FB9,0x1FB2,0x1FBC,
+        0x1FCC,0x0001,0x1FC3,
+        0x1FD0,0x0602,
+        0x1FE0,0x0602,
+        0x1FE5,0x0001,0x1FEC,
+        0x1FF3,0x0001,0x1FFC,
         /* Letterlike Symbols */
         0x214E,0x0001,0x2132,
         /* Number forms */
-        0x2170,0x0210,  0x2184,0x0001,0x2183,
+        0x2170,0x0210,
+        0x2184,0x0001,0x2183,
         /* Enclosed Alphanumerics */
-        0x24D0,0x051A,  0x2C30,0x042F,
+        0x24D0,0x051A,
+        0x2C30,0x042F,
         /* Latin Extended-C */
-        0x2C60,0x0102,  0x2C67,0x0106, 0x2C75,0x0102,
+        0x2C60,0x0102,
+        0x2C67,0x0106, 0x2C75,0x0102,
         /* Coptic */
         0x2C80,0x0164,
         /* Georgian Supplement */
@@ -355,34 +590,38 @@ WCHAR ff_wtoupper ( /* Returns upper converted character */
         /* Full-width */
         0xFF41,0x031A,
 
-        0x0000
+        0x0000  /* EOT */
     };
-    const WCHAR *p;
-    WCHAR bc, nc, cmd;
 
 
-    p = chr < 0x1000 ? cvt1 : cvt2;
-    for (;;) {
-        bc = *p++;                              /* Get block base */
-        if (!bc || chr < bc) break;
-        nc = *p++; cmd = nc >> 8; nc &= 0xFF;   /* Get processing command and block size */
-        if (chr < bc + nc) {    /* In the block? */
-            switch (cmd) {
-            case 0: chr = p[chr - bc]; break;       /* Table conversion */
-            case 1: chr -= (chr - bc) & 1; break;   /* Case pairs */
-            case 2: chr -= 16; break;               /* Shift -16 */
-            case 3: chr -= 32; break;               /* Shift -32 */
-            case 4: chr -= 48; break;               /* Shift -48 */
-            case 5: chr -= 26; break;               /* Shift -26 */
-            case 6: chr += 8; break;                /* Shift +8 */
-            case 7: chr -= 80; break;               /* Shift -80 */
-            case 8: chr -= 0x1C60; break;           /* Shift -0x1C60 */
+    if (uni < 0x10000) {    /* Is it in BMP? */
+        uc = (WORD)uni;
+        p = uc < 0x1000 ? cvt1 : cvt2;
+        for (;;) {
+            bc = *p++;                              /* Get the block base */
+            if (bc == 0 || uc < bc) break;          /* Not matched? */
+            nc = *p++; cmd = nc >> 8; nc &= 0xFF;   /* Get processing command and block size */
+            if (uc < bc + nc) { /* In the block? */
+                switch (cmd) {
+                case 0: uc = p[uc - bc]; break;     /* Table conversion */
+                case 1: uc -= (uc - bc) & 1; break; /* Case pairs */
+                case 2: uc -= 16; break;            /* Shift -16 */
+                case 3: uc -= 32; break;            /* Shift -32 */
+                case 4: uc -= 48; break;            /* Shift -48 */
+                case 5: uc -= 26; break;            /* Shift -26 */
+                case 6: uc += 8; break;             /* Shift +8 */
+                case 7: uc -= 80; break;            /* Shift -80 */
+                case 8: uc -= 0x1C60; break;        /* Shift -0x1C60 */
+                }
+                break;
             }
-            break;
+            if (cmd == 0) p += nc;  /* Skip table if needed */
         }
-        if (!cmd) p += nc;
+        uni = uc;
     }
 
-    return chr;
+    return uni;
 }
 
+
+#endif /* #if FF_USE_LFN */

lib/oofatfs/option/unicode.c

@@ -1,17 +0,0 @@
-#include "../ff.h"
-
-#if _USE_LFN != 0
-
-#if   _CODE_PAGE == 932 /* Japanese Shift_JIS */
-#include "cc932.c"
-#elif _CODE_PAGE == 936 /* Simplified Chinese GBK */
-#include "cc936.c"
-#elif _CODE_PAGE == 949 /* Korean */
-#include "cc949.c"
-#elif _CODE_PAGE == 950 /* Traditional Chinese Big5 */
-#include "cc950.c"
-#else                   /* Single Byte Character-Set */
-#include "ccsbcs.c"
-#endif
-
-#endif

lib/utils/gchelper.h

@@ -3,8 +3,7 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2013, 2014 Damien P. George
- * Copyright (c) 2015 Daniel Campora
+ * Copyright (c) 2019 Damien P. George
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -24,10 +23,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#ifndef MICROPY_INCLUDED_CC3200_UTIL_GCHELPER_H
-#define MICROPY_INCLUDED_CC3200_UTIL_GCHELPER_H
+#ifndef MICROPY_INCLUDED_LIB_UTILS_GCHELPER_H
+#define MICROPY_INCLUDED_LIB_UTILS_GCHELPER_H
 
-extern mp_uint_t gc_helper_get_sp(void);
-extern mp_uint_t gc_helper_get_regs_and_sp(mp_uint_t *regs);
+#include <stdint.h>
 
-#endif // MICROPY_INCLUDED_CC3200_UTIL_GCHELPER_H
+uintptr_t gc_helper_get_sp(void);
+uintptr_t gc_helper_get_regs_and_sp(uintptr_t *regs);
+
+#endif // MICROPY_INCLUDED_LIB_UTILS_GCHELPER_H

lib/utils/gchelper_m3.s

@@ -0,0 +1,67 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2013-2014 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+    .syntax unified
+    .cpu cortex-m3
+    .thumb
+
+    .section .text
+    .align  2
+
+    .global gc_helper_get_sp
+    .type gc_helper_get_sp, %function
+
+@ uint gc_helper_get_sp(void)
+gc_helper_get_sp:
+    @ return the sp
+    mov     r0, sp
+    bx      lr
+
+    .size gc_helper_get_sp, .-gc_helper_get_sp
+
+
+    .global gc_helper_get_regs_and_sp
+    .type gc_helper_get_regs_and_sp, %function
+
+@ uint gc_helper_get_regs_and_sp(r0=uint regs[10])
+gc_helper_get_regs_and_sp:
+    @ store registers into given array
+    str     r4, [r0], #4
+    str     r5, [r0], #4
+    str     r6, [r0], #4
+    str     r7, [r0], #4
+    str     r8, [r0], #4
+    str     r9, [r0], #4
+    str     r10, [r0], #4
+    str     r11, [r0], #4
+    str     r12, [r0], #4
+    str     r13, [r0], #4
+
+    @ return the sp
+    mov     r0, sp
+    bx      lr
+
+    .size gc_helper_get_regs_and_sp, .-gc_helper_get_regs_and_sp

lib/utils/interrupt_char.c

@@ -29,7 +29,7 @@
 
 #if MICROPY_KBD_EXCEPTION
 
-int mp_interrupt_char;
+int mp_interrupt_char = -1;
 
 void mp_hal_set_interrupt_char(int c) {
     if (c != -1) {

lib/utils/printf.c

@@ -99,9 +99,11 @@ STATIC void strn_print_strn(void *data, const char *str, size_t len) {
     strn_print_env->remain -= len;
 }
 
-#if defined(__GNUC__) && !defined(__clang__)
+#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 9
 // uClibc requires this alias to be defined, or there may be link errors
 // when linkings against it statically.
+// GCC 9 gives a warning about missing attributes so it's excluded until
+// uClibc+GCC9 support is needed.
 int __GI_vsnprintf(char *str, size_t size, const char *fmt, va_list ap) __attribute__((weak, alias ("vsnprintf")));
 #endif
 

lib/utils/pyexec.c

@@ -160,6 +160,7 @@ typedef struct _repl_t {
     // will be added later.
     //vstr_t line;
     bool cont_line;
+    bool paste_mode;
 } repl_t;
 
 repl_t repl;
@@ -170,6 +171,7 @@ STATIC int pyexec_friendly_repl_process_char(int c);
 void pyexec_event_repl_init(void) {
     MP_STATE_VM(repl_line) = vstr_new(32);
     repl.cont_line = false;
+    repl.paste_mode = false;
     // no prompt before printing friendly REPL banner or entering raw REPL
     readline_init(MP_STATE_VM(repl_line), "");
     if (pyexec_mode_kind == PYEXEC_MODE_RAW_REPL) {
@@ -189,6 +191,7 @@ STATIC int pyexec_raw_repl_process_char(int c) {
         pyexec_mode_kind = PYEXEC_MODE_FRIENDLY_REPL;
         vstr_reset(MP_STATE_VM(repl_line));
         repl.cont_line = false;
+        repl.paste_mode = false;
         pyexec_friendly_repl_process_char(CHAR_CTRL_B);
         return 0;
     } else if (c == CHAR_CTRL_C) {
@@ -226,6 +229,32 @@ reset:
 }
 
 STATIC int pyexec_friendly_repl_process_char(int c) {
+    if (repl.paste_mode) {
+        if (c == CHAR_CTRL_C) {
+            // cancel everything
+            mp_hal_stdout_tx_str("\r\n");
+            goto input_restart;
+        } else if (c == CHAR_CTRL_D) {
+            // end of input
+            mp_hal_stdout_tx_str("\r\n");
+            int ret = parse_compile_execute(MP_STATE_VM(repl_line), MP_PARSE_FILE_INPUT, EXEC_FLAG_ALLOW_DEBUGGING | EXEC_FLAG_IS_REPL | EXEC_FLAG_SOURCE_IS_VSTR);
+            if (ret & PYEXEC_FORCED_EXIT) {
+                return ret;
+            }
+            goto input_restart;
+        } else {
+            // add char to buffer and echo
+            vstr_add_byte(MP_STATE_VM(repl_line), c);
+            if (c == '\r') {
+                mp_hal_stdout_tx_str("\r\n=== ");
+            } else {
+                char buf[1] = {c};
+                mp_hal_stdout_tx_strn(buf, 1);
+            }
+            return 0;
+        }
+    }
+
     int ret = readline_process_char(c);
 
     if (!repl.cont_line) {
@@ -253,6 +282,12 @@ STATIC int pyexec_friendly_repl_process_char(int c) {
             mp_hal_stdout_tx_str("\r\n");
             vstr_clear(MP_STATE_VM(repl_line));
             return PYEXEC_FORCED_EXIT;
+        } else if (ret == CHAR_CTRL_E) {
+            // paste mode
+            mp_hal_stdout_tx_str("\r\npaste mode; Ctrl-C to cancel, Ctrl-D to finish\r\n=== ");
+            vstr_reset(MP_STATE_VM(repl_line));
+            repl.paste_mode = true;
+            return 0;
         }
 
         if (ret < 0) {
@@ -299,6 +334,7 @@ exec: ;
 input_restart:
         vstr_reset(MP_STATE_VM(repl_line));
         repl.cont_line = false;
+        repl.paste_mode = false;
         readline_init(MP_STATE_VM(repl_line), ">>> ");
         return 0;
     }
@@ -414,7 +450,7 @@ friendly_repl_reset:
             // do the user a favor and reenable interrupts.
             if (query_irq() == IRQ_STATE_DISABLED) {
                 enable_irq(IRQ_STATE_ENABLED);
-                mp_hal_stdout_tx_str("PYB: enabling IRQs\r\n");
+                mp_hal_stdout_tx_str("MPY: enabling IRQs\r\n");
             }
         }
         #endif
@@ -505,6 +541,18 @@ int pyexec_file(const char *filename) {
     return parse_compile_execute(filename, MP_PARSE_FILE_INPUT, EXEC_FLAG_SOURCE_IS_FILENAME);
 }
 
+int pyexec_file_if_exists(const char *filename) {
+    #if MICROPY_MODULE_FROZEN
+    if (mp_frozen_stat(filename) == MP_IMPORT_STAT_FILE) {
+        return pyexec_frozen_module(filename);
+    }
+    #endif
+    if (mp_import_stat(filename) != MP_IMPORT_STAT_FILE) {
+        return 1; // success (no file is the same as an empty file executing without fail)
+    }
+    return pyexec_file(filename);
+}
+
 #if MICROPY_MODULE_FROZEN
 int pyexec_frozen_module(const char *name) {
     void *frozen_data;

lib/utils/pyexec.h

@@ -46,6 +46,7 @@ extern int pyexec_system_exit;
 int pyexec_raw_repl(void);
 int pyexec_friendly_repl(void);
 int pyexec_file(const char *filename);
+int pyexec_file_if_exists(const char *filename);
 int pyexec_frozen_module(const char *name);
 void pyexec_event_repl_init(void);
 int pyexec_event_repl_process_char(int c);

mpy-cross/Makefile

@@ -9,6 +9,8 @@ override undefine MICROPY_FORCE_32BIT
 override undefine CROSS_COMPILE
 override undefine FROZEN_DIR
 override undefine FROZEN_MPY_DIR
+override undefine USER_C_MODULES
+override undefine SRC_MOD
 override undefine BUILD
 override undefine PROG
 endif

mpy-cross/gccollect.c

@@ -143,9 +143,6 @@ void gc_collect(void) {
     // GC stack (and regs because we captured them)
     void **regs_ptr = (void**)(void*)&regs;
     gc_collect_root(regs_ptr, ((mp_uint_t)MP_STATE_THREAD(stack_top) - (mp_uint_t)&regs) / sizeof(mp_uint_t));
-    #if MICROPY_EMIT_NATIVE
-    mp_unix_mark_exec();
-    #endif
     gc_collect_end();
 }
 

mpy-cross/main.c

@@ -34,6 +34,7 @@
 #include "py/runtime.h"
 #include "py/gc.h"
 #include "py/stackctrl.h"
+#include "genhdr/mpversion.h"
 #ifdef _WIN32
 #include "ports/windows/fmode.h"
 #endif
@@ -67,9 +68,7 @@ STATIC int compile_and_save(const char *file, const char *output_file, const cha
         }
 
         #if MICROPY_PY___FILE__
-        if (input_kind == MP_PARSE_FILE_INPUT) {
-            mp_store_global(MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name));
-        }
+        mp_store_global(MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name));
         #endif
 
         mp_parse_tree_t parse_tree = mp_parse(lex, MP_PARSE_FILE_INPUT);
@@ -100,6 +99,7 @@ STATIC int usage(char **argv) {
     printf(
 "usage: %s [<opts>] [-X <implopt>] <input filename>\n"
 "Options:\n"
+"--version : show version information\n"
 "-o : output file for compiled bytecode (defaults to input with .mpy extension)\n"
 "-s : source filename to embed in the compiled bytecode (defaults to input file)\n"
 "-v : verbose (trace various operations); can be multiple\n"
@@ -109,6 +109,7 @@ STATIC int usage(char **argv) {
 "-msmall-int-bits=number : set the maximum bits used to encode a small-int\n"
 "-mno-unicode : don't support unicode in compiled strings\n"
 "-mcache-lookup-bc : cache map lookups in the bytecode\n"
+"-march=<arch> : set architecture for native emitter; x86, x64, armv6, armv7m, xtensa\n"
 "\n"
 "Implementation specific options:\n", argv[0]
 );
@@ -193,6 +194,15 @@ MP_NOINLINE int main_(int argc, char **argv) {
     mp_dynamic_compiler.small_int_bits = 31;
     mp_dynamic_compiler.opt_cache_map_lookup_in_bytecode = 0;
     mp_dynamic_compiler.py_builtins_str_unicode = 1;
+    #if defined(__i386__)
+    mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_X86;
+    #elif defined(__x86_64__)
+    mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_X64;
+    #elif defined(__arm__) && !defined(__thumb2__)
+    mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_ARMV6;
+    #else
+    mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_NONE;
+    #endif
 
     const char *input_file = NULL;
     const char *output_file = NULL;
@@ -203,6 +213,10 @@ MP_NOINLINE int main_(int argc, char **argv) {
         if (argv[a][0] == '-') {
             if (strcmp(argv[a], "-X") == 0) {
                 a += 1;
+            } else if (strcmp(argv[a], "--version") == 0) {
+                printf("MicroPython " MICROPY_GIT_TAG " on " MICROPY_BUILD_DATE
+                    "; mpy-cross emitting mpy v" MP_STRINGIFY(MPY_VERSION) "\n");
+                return 0;
             } else if (strcmp(argv[a], "-v") == 0) {
                 mp_verbose_flag++;
             } else if (strncmp(argv[a], "-O", 2) == 0) {
@@ -240,6 +254,21 @@ MP_NOINLINE int main_(int argc, char **argv) {
                 mp_dynamic_compiler.py_builtins_str_unicode = 0;
             } else if (strcmp(argv[a], "-municode") == 0) {
                 mp_dynamic_compiler.py_builtins_str_unicode = 1;
+            } else if (strncmp(argv[a], "-march=", sizeof("-march=") - 1) == 0) {
+                const char *arch = argv[a] + sizeof("-march=") - 1;
+                if (strcmp(arch, "x86") == 0) {
+                    mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_X86;
+                } else if (strcmp(arch, "x64") == 0) {
+                    mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_X64;
+                } else if (strcmp(arch, "armv6") == 0) {
+                    mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_ARMV6;
+                } else if (strcmp(arch, "armv7m") == 0) {
+                    mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_ARMV7M;
+                } else if (strcmp(arch, "xtensa") == 0) {
+                    mp_dynamic_compiler.native_arch = MP_NATIVE_ARCH_XTENSA;
+                } else {
+                    return usage(argv);
+                }
             } else {
                 return usage(argv);
             }

mpy-cross/mpconfigport.h

@@ -30,13 +30,15 @@
 #define MICROPY_PERSISTENT_CODE_LOAD (0)
 #define MICROPY_PERSISTENT_CODE_SAVE (1)
 
-#define MICROPY_EMIT_X64            (0)
-#define MICROPY_EMIT_X86            (0)
-#define MICROPY_EMIT_THUMB          (0)
-#define MICROPY_EMIT_INLINE_THUMB   (0)
-#define MICROPY_EMIT_INLINE_THUMB_ARMV7M (0)
-#define MICROPY_EMIT_INLINE_THUMB_FLOAT (0)
-#define MICROPY_EMIT_ARM            (0)
+#define MICROPY_EMIT_X64            (1)
+#define MICROPY_EMIT_X86            (1)
+#define MICROPY_EMIT_THUMB          (1)
+#define MICROPY_EMIT_INLINE_THUMB   (1)
+#define MICROPY_EMIT_INLINE_THUMB_ARMV7M (1)
+#define MICROPY_EMIT_INLINE_THUMB_FLOAT (1)
+#define MICROPY_EMIT_ARM            (1)
+#define MICROPY_EMIT_XTENSA         (1)
+#define MICROPY_EMIT_INLINE_XTENSA  (1)
 
 #define MICROPY_DYNAMIC_COMPILER    (1)
 #define MICROPY_COMP_CONST_FOLDING  (1)
@@ -110,10 +112,6 @@ typedef long mp_off_t;
 
 #define MP_PLAT_PRINT_STRN(str, len) (void)0
 
-#ifndef MP_NOINLINE
-#define MP_NOINLINE __attribute__((noinline))
-#endif
-
 // We need to provide a declaration/definition of alloca()
 #ifdef __FreeBSD__
 #include <stdlib.h>

ports/cc3200/application.mk

@@ -128,7 +128,6 @@ APP_UTIL_SRC_C = $(addprefix util/,\
 	)
 	
 APP_UTIL_SRC_S = $(addprefix util/,\
-	gchelper.s \
 	sleeprestore.s \
 	)
 	
@@ -141,7 +140,7 @@ APP_MAIN_SRC_C = \
 	
 APP_LIB_SRC_C = $(addprefix lib/,\
 	oofatfs/ff.c \
-	oofatfs/option/unicode.c \
+	oofatfs/ffunicode.c \
 	libc/string0.c \
 	mp-readline/readline.c \
 	netutils/netutils.c \
@@ -159,6 +158,7 @@ APP_STM_SRC_C = $(addprefix ports/stm32/,\
 OBJ = $(PY_O) $(addprefix $(BUILD)/, $(APP_FATFS_SRC_C:.c=.o) $(APP_RTOS_SRC_C:.c=.o) $(APP_FTP_SRC_C:.c=.o) $(APP_HAL_SRC_C:.c=.o) $(APP_MISC_SRC_C:.c=.o))
 OBJ += $(addprefix $(BUILD)/, $(APP_MODS_SRC_C:.c=.o) $(APP_CC3100_SRC_C:.c=.o) $(APP_SL_SRC_C:.c=.o) $(APP_TELNET_SRC_C:.c=.o) $(APP_UTIL_SRC_C:.c=.o) $(APP_UTIL_SRC_S:.s=.o))
 OBJ += $(addprefix $(BUILD)/, $(APP_MAIN_SRC_C:.c=.o) $(APP_LIB_SRC_C:.c=.o) $(APP_STM_SRC_C:.c=.o))
+OBJ += $(BUILD)/lib/utils/gchelper_m3.o
 OBJ += $(BUILD)/pins.o
 
 # List of sources for qstr extraction

ports/cc3200/fatfs_port.c

@@ -30,12 +30,12 @@
 #include "lib/timeutils/timeutils.h"
 #include "mods/pybrtc.h"
 
-#if _FS_REENTRANT
+#if FF_FS_REENTRANT
 // Create a Synchronization Object
 // This function is called in f_mount() function to create a new
 // synchronization object, such as semaphore and mutex.
 // A return of 0 indicates failure, and then f_mount() fails with FR_INT_ERR.
-int ff_cre_syncobj(FATFS *fatfs, _SYNC_t *sobj) {
+int ff_cre_syncobj(FATFS *fatfs, FF_SYNC_t *sobj) {
     vSemaphoreCreateBinary((*sobj));
     return (int)(*sobj != NULL);
 }
@@ -44,7 +44,7 @@ int ff_cre_syncobj(FATFS *fatfs, _SYNC_t *sobj) {
 // This function is called in f_mount() function to delete a synchronization
 // object that created with ff_cre_syncobj function.
 // A return of 0 indicates failure, and then f_mount() fails with FR_INT_ERR.
-int ff_del_syncobj(_SYNC_t sobj) {
+int ff_del_syncobj(FF_SYNC_t sobj) {
     vSemaphoreDelete(sobj);
     return 1;
 }
@@ -52,13 +52,13 @@ int ff_del_syncobj(_SYNC_t sobj) {
 // Request Grant to Access the Volume
 // This function is called on entering file functions to lock the volume.
 // When a 0 is returned, the file function fails with FR_TIMEOUT.
-int ff_req_grant(_SYNC_t sobj) {
-    return (int)(xSemaphoreTake(sobj, _FS_TIMEOUT) == pdTRUE);
+int ff_req_grant(FF_SYNC_t sobj) {
+    return (int)(xSemaphoreTake(sobj, FF_FS_TIMEOUT) == pdTRUE);
 }
 
 // Release Grant to Access the Volume
 // This function is called on leaving file functions to unlock the volume.
-void ff_rel_grant(_SYNC_t sobj) {
+void ff_rel_grant(FF_SYNC_t sobj) {
     xSemaphoreGive(sobj);
 }
 

ports/cc3200/hal/cc3200_hal.c

@@ -148,7 +148,7 @@ void mp_hal_stdout_tx_str(const char *str) {
 
 void mp_hal_stdout_tx_strn(const char *str, size_t len) {
     if (MP_STATE_PORT(os_term_dup_obj)) {
-        if (MP_OBJ_IS_TYPE(MP_STATE_PORT(os_term_dup_obj)->stream_o, &pyb_uart_type)) {
+        if (mp_obj_is_type(MP_STATE_PORT(os_term_dup_obj)->stream_o, &pyb_uart_type)) {
             uart_tx_strn(MP_STATE_PORT(os_term_dup_obj)->stream_o, str, len);
         } else {
             MP_STATE_PORT(os_term_dup_obj)->write[2] = mp_obj_new_str_of_type(&mp_type_str, (const byte *)str, len);
@@ -184,7 +184,7 @@ int mp_hal_stdin_rx_chr(void) {
         if (telnet_rx_any()) {
             return telnet_rx_char();
         } else if (MP_STATE_PORT(os_term_dup_obj)) { // then the stdio_dup
-            if (MP_OBJ_IS_TYPE(MP_STATE_PORT(os_term_dup_obj)->stream_o, &pyb_uart_type)) {
+            if (mp_obj_is_type(MP_STATE_PORT(os_term_dup_obj)->stream_o, &pyb_uart_type)) {
                 if (uart_rx_any(MP_STATE_PORT(os_term_dup_obj)->stream_o)) {
                     return uart_rx_char(MP_STATE_PORT(os_term_dup_obj)->stream_o);
                 }

ports/cc3200/mods/modmachine.c

@@ -122,7 +122,7 @@ STATIC mp_obj_t machine_unique_id(void) {
 STATIC MP_DEFINE_CONST_FUN_OBJ_0(machine_unique_id_obj, machine_unique_id);
 
 STATIC mp_obj_t machine_main(mp_obj_t main) {
-    if (MP_OBJ_IS_STR(main)) {
+    if (mp_obj_is_str(main)) {
         MP_STATE_PORT(machine_config_main) = main;
     } else {
         mp_raise_ValueError(mpexception_value_invalid_arguments);
@@ -137,11 +137,11 @@ STATIC mp_obj_t machine_idle(void) {
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_0(machine_idle_obj, machine_idle);
 
-STATIC mp_obj_t machine_sleep (void) {
+STATIC mp_obj_t machine_lightsleep(void) {
     pyb_sleep_sleep();
     return mp_const_none;
 }
-STATIC MP_DEFINE_CONST_FUN_OBJ_0(machine_sleep_obj, machine_sleep);
+STATIC MP_DEFINE_CONST_FUN_OBJ_0(machine_lightsleep_obj, machine_lightsleep);
 
 STATIC mp_obj_t machine_deepsleep (void) {
     pyb_sleep_deepsleep();
@@ -171,7 +171,8 @@ STATIC const mp_rom_map_elem_t machine_module_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR_main),                MP_ROM_PTR(&machine_main_obj) },
     { MP_ROM_QSTR(MP_QSTR_rng),                 MP_ROM_PTR(&machine_rng_get_obj) },
     { MP_ROM_QSTR(MP_QSTR_idle),                MP_ROM_PTR(&machine_idle_obj) },
-    { MP_ROM_QSTR(MP_QSTR_sleep),               MP_ROM_PTR(&machine_sleep_obj) },
+    { MP_ROM_QSTR(MP_QSTR_sleep),               MP_ROM_PTR(&machine_lightsleep_obj) },
+    { MP_ROM_QSTR(MP_QSTR_lightsleep),          MP_ROM_PTR(&machine_lightsleep_obj) },
     { MP_ROM_QSTR(MP_QSTR_deepsleep),           MP_ROM_PTR(&machine_deepsleep_obj) },
     { MP_ROM_QSTR(MP_QSTR_reset_cause),         MP_ROM_PTR(&machine_reset_cause_obj) },
     { MP_ROM_QSTR(MP_QSTR_wake_reason),         MP_ROM_PTR(&machine_wake_reason_obj) },

ports/cc3200/mods/moduos.c

@@ -133,7 +133,7 @@ STATIC mp_obj_t os_dupterm(uint n_args, const mp_obj_t *args) {
         if (stream_o == mp_const_none) {
             MP_STATE_PORT(os_term_dup_obj) = MP_OBJ_NULL;
         } else {
-            if (!MP_OBJ_IS_TYPE(stream_o, &pyb_uart_type)) {
+            if (!mp_obj_is_type(stream_o, &pyb_uart_type)) {
                 // must be a stream-like object providing at least read and write methods
                 mp_load_method(stream_o, MP_QSTR_read, os_term_dup_obj.read);
                 mp_load_method(stream_o, MP_QSTR_write, os_term_dup_obj.write);

ports/cc3200/mods/modussl.c

@@ -85,8 +85,8 @@ STATIC mp_obj_t mod_ssl_wrap_socket(size_t n_args, const mp_obj_t *pos_args, mp_
     mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
     mp_arg_parse_all(n_args, pos_args, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
 
-    // chech if ca validation is required
-    if (args[4].u_int != SSL_CERT_NONE && args[5].u_obj == mp_const_none) {
+    // check if ca validation is required
+    if (args[4].u_int != SSL_CERT_NONE && args[6].u_obj == mp_const_none) {
         goto arg_error;
     }
 

ports/cc3200/mods/modwlan.c

@@ -1012,7 +1012,7 @@ STATIC mp_obj_t wlan_ifconfig(size_t n_args, const mp_obj_t *pos_args, mp_map_t
         };
         return mp_obj_new_tuple(4, ifconfig);
     } else { // set the configuration
-        if (MP_OBJ_IS_TYPE(args[1].u_obj, &mp_type_tuple)) {
+        if (mp_obj_is_type(args[1].u_obj, &mp_type_tuple)) {
             // set a static ip
             mp_obj_t *items;
             mp_obj_get_array_fixed_n(args[1].u_obj, 4, &items);

ports/cc3200/mods/pybpin.c

@@ -131,7 +131,7 @@ pin_obj_t *pin_find(mp_obj_t user_obj) {
     pin_obj_t *pin_obj;
 
     // if a pin was provided, use it
-    if (MP_OBJ_IS_TYPE(user_obj, &pin_type)) {
+    if (mp_obj_is_type(user_obj, &pin_type)) {
         pin_obj = user_obj;
         return pin_obj;
     }

ports/cc3200/mods/pybrtc.c

@@ -367,7 +367,7 @@ STATIC mp_obj_t pyb_rtc_alarm(size_t n_args, const mp_obj_t *pos_args, mp_map_t
     uint32_t f_seconds;
     uint16_t f_mseconds;
     bool repeat = args[2].u_bool;
-    if (MP_OBJ_IS_TYPE(args[1].u_obj, &mp_type_tuple)) { // datetime tuple given
+    if (mp_obj_is_type(args[1].u_obj, &mp_type_tuple)) { // datetime tuple given
         // repeat cannot be used with a datetime tuple
         if (repeat) {
             mp_raise_ValueError(mpexception_value_invalid_arguments);

ports/cc3200/mpconfigport.h

@@ -39,6 +39,7 @@
 #define MICROPY_PERSISTENT_CODE_LOAD                (1)
 #define MICROPY_EMIT_THUMB                          (0)
 #define MICROPY_EMIT_INLINE_THUMB                   (0)
+#define MICROPY_COMP_CONST_LITERAL                  (0)
 #define MICROPY_COMP_MODULE_CONST                   (1)
 #define MICROPY_ENABLE_GC                           (1)
 #define MICROPY_ENABLE_FINALISER                    (1)
@@ -64,7 +65,7 @@
 // fatfs configuration used in ffconf.h
 #define MICROPY_FATFS_ENABLE_LFN                    (2)
 #define MICROPY_FATFS_MAX_LFN                       (MICROPY_ALLOC_PATH_MAX)
-#define MICROPY_FATFS_LFN_CODE_PAGE                 (437) // 1=SFN/ANSI 437=LFN/U.S.(OEM)
+#define MICROPY_FATFS_LFN_CODE_PAGE                 437 // 1=SFN/ANSI 437=LFN/U.S.(OEM)
 #define MICROPY_FATFS_RPATH                         (2)
 #define MICROPY_FATFS_REENTRANT                     (1)
 #define MICROPY_FATFS_TIMEOUT                       (2500)

ports/cc3200/mptask.c

@@ -50,8 +50,8 @@
 #include "pybpin.h"
 #include "pybrtc.h"
 #include "lib/utils/pyexec.h"
+#include "lib/utils/gchelper.h"
 #include "gccollect.h"
-#include "gchelper.h"
 #include "mperror.h"
 #include "simplelink.h"
 #include "modnetwork.h"
@@ -180,7 +180,7 @@ soft_reset:
 
     if (!safeboot) {
         // run boot.py
-        int ret = pyexec_file("boot.py");
+        int ret = pyexec_file_if_exists("boot.py");
         if (ret & PYEXEC_FORCED_EXIT) {
             goto soft_reset_exit;
         }
@@ -205,7 +205,7 @@ soft_reset:
             } else {
                 main_py = mp_obj_str_get_str(MP_STATE_PORT(machine_config_main));
             }
-            int ret = pyexec_file(main_py);
+            int ret = pyexec_file_if_exists(main_py);
             if (ret & PYEXEC_FORCED_EXIT) {
                 goto soft_reset_exit;
             }
@@ -234,7 +234,7 @@ soft_reset_exit:
 
     // soft reset
     pyb_sleep_signal_soft_reset();
-    mp_printf(&mp_plat_print, "PYB: soft reboot\n");
+    mp_printf(&mp_plat_print, "MPY: soft reboot\n");
 
     // disable all callbacks to avoid undefined behaviour
     // when coming out of a soft reset
@@ -307,7 +307,7 @@ STATIC void mptask_init_sflash_filesystem (void) {
     FRESULT res = f_mount(&vfs_fat->fatfs);
     if (res == FR_NO_FILESYSTEM) {
         // no filesystem, so create a fresh one
-        uint8_t working_buf[_MAX_SS];
+        uint8_t working_buf[FF_MAX_SS];
         res = f_mkfs(&vfs_fat->fatfs, FM_FAT | FM_SFD, 0, working_buf, sizeof(working_buf));
         if (res == FR_OK) {
             // success creating fresh LFS

ports/cc3200/util/gccollect.c

@@ -30,8 +30,8 @@
 
 #include "py/gc.h"
 #include "py/mpthread.h"
+#include "lib/utils/gchelper.h"
 #include "gccollect.h"
-#include "gchelper.h"
 
 /******************************************************************************
 DECLARE PUBLIC FUNCTIONS

ports/cc3200/util/gchelper.s

@@ -1,41 +0,0 @@
-    .syntax unified
-    .cpu cortex-m4
-    .thumb
-    .text
-    .align  2
-
-
-
-@ uint gc_helper_get_sp(void)
-    .global gc_helper_get_sp
-    .thumb
-    .thumb_func
-    .type   gc_helper_get_sp, %function
-gc_helper_get_sp:
-    @ return the sp
-    mov     r0, sp
-    bx      lr
-
-
-
-@ uint gc_helper_get_regs_and_sp(r0=uint regs[10])
-    .global gc_helper_get_regs_and_sp
-    .thumb
-    .thumb_func
-    .type   gc_helper_get_regs_and_sp, %function
-gc_helper_get_regs_and_sp:
-    @ store registers into given array
-    str     r4, [r0], #4
-    str     r5, [r0], #4
-    str     r6, [r0], #4
-    str     r7, [r0], #4
-    str     r8, [r0], #4
-    str     r9, [r0], #4
-    str     r10, [r0], #4
-    str     r11, [r0], #4
-    str     r12, [r0], #4
-    str     r13, [r0], #4
-
-    @ return the sp
-    mov     r0, sp
-    bx      lr

ports/esp32/Makefile

@@ -20,8 +20,14 @@ FLASH_MODE ?= dio
 FLASH_FREQ ?= 40m
 FLASH_SIZE ?= 4MB
 CROSS_COMPILE ?= xtensa-esp32-elf-
+OBJDUMP = $(CROSS_COMPILE)objdump
 
-ESPIDF_SUPHASH := 30545f4cccec7460634b656d278782dd7151098e
+# SDKCONFIG should be overridden to get a different configuration
+SDKCONFIG ?= boards/sdkconfig
+SDKCONFIG_H = $(BUILD)/sdkconfig.h
+
+# the git hash of the currently supported ESP IDF version
+ESPIDF_SUPHASH := 5c88c5996dbde6208e3bec05abc21ff6cd822d26
 
 # paths to ESP IDF and its components
 ifeq ($(ESPIDF),)
@@ -36,6 +42,8 @@ endif
 endif
 ESPCOMP = $(ESPIDF)/components
 ESPTOOL ?= $(ESPCOMP)/esptool_py/esptool/esptool.py
+ESPCOMP_KCONFIGS = $(shell find $(ESPCOMP) -name Kconfig)
+ESPCOMP_KCONFIGS_PROJBUILD = $(shell find $(ESPCOMP) -name Kconfig.projbuild)
 
 # verify the ESP IDF version
 ESPIDF_CURHASH := $(shell git -C $(ESPIDF) show -s --pretty=format:'%H')
@@ -51,6 +59,12 @@ endif
 # pretty format of ESP IDF version, used internally by the IDF
 IDF_VER := $(shell git -C $(ESPIDF) describe)
 
+# include sdkconfig to get needed configuration values
+include $(SDKCONFIG)
+
+################################################################################
+# Compiler and linker flags
+
 INC += -I.
 INC += -I$(TOP)
 INC += -I$(TOP)/lib/mp-readline
@@ -78,10 +92,12 @@ INC_ESPCOMP += -I$(ESPCOMP)/newlib/platform_include
 INC_ESPCOMP += -I$(ESPCOMP)/newlib/include
 INC_ESPCOMP += -I$(ESPCOMP)/nvs_flash/include
 INC_ESPCOMP += -I$(ESPCOMP)/freertos/include
+INC_ESPCOMP += -I$(ESPCOMP)/esp_ringbuf/include
+INC_ESPCOMP += -I$(ESPCOMP)/esp_event/include
 INC_ESPCOMP += -I$(ESPCOMP)/tcpip_adapter/include
-INC_ESPCOMP += -I$(ESPCOMP)/lwip/include/lwip
-INC_ESPCOMP += -I$(ESPCOMP)/lwip/include/lwip/port
-INC_ESPCOMP += -I$(ESPCOMP)/lwip/include/lwip/posix
+INC_ESPCOMP += -I$(ESPCOMP)/lwip/lwip/src/include
+INC_ESPCOMP += -I$(ESPCOMP)/lwip/port/esp32/include
+INC_ESPCOMP += -I$(ESPCOMP)/lwip/include/apps
 INC_ESPCOMP += -I$(ESPCOMP)/mbedtls/mbedtls/include
 INC_ESPCOMP += -I$(ESPCOMP)/mbedtls/port/include
 INC_ESPCOMP += -I$(ESPCOMP)/spi_flash/include
@@ -117,8 +133,9 @@ LDFLAGS += -u call_user_start_cpu0 -u uxTopUsedPriority -u ld_include_panic_high
 LDFLAGS += -u __cxa_guard_dummy # so that implementation of static guards is taken from cxx_guards.o instead of libstdc++.a
 LDFLAGS += -L$(ESPCOMP)/esp32/ld
 LDFLAGS += -T $(BUILD)/esp32_out.ld
-LDFLAGS += -T ./esp32.custom_common.ld
+LDFLAGS += -T $(BUILD)/esp32.common.ld
 LDFLAGS += -T esp32.rom.ld
+LDFLAGS += -T esp32.rom.libgcc.ld
 LDFLAGS += -T esp32.peripherals.ld
 
 LIBGCC_FILE_NAME = $(shell $(CC) $(CFLAGS) -print-libgcc-file-name)
@@ -134,9 +151,9 @@ COPT += -Os -DNDEBUG
 #LDFLAGS += --gc-sections
 endif
 
-# Enable SPIRAM support if CONFIG_SPIRAM_SUPPORT=1
-ifeq ($(CONFIG_SPIRAM_SUPPORT),1)
-CFLAGS_COMMON += -mfix-esp32-psram-cache-issue -DCONFIG_SPIRAM_SUPPORT=1
+# Enable SPIRAM support if CONFIG_SPIRAM_SUPPORT=y in sdkconfig
+ifeq ($(CONFIG_SPIRAM_SUPPORT),y)
+CFLAGS_COMMON += -mfix-esp32-psram-cache-issue
 LIBC_LIBM = $(ESPCOMP)/newlib/lib/libc-psram-workaround.a $(ESPCOMP)/newlib/lib/libm-psram-workaround.a
 else
 LDFLAGS += -T esp32.rom.spiram_incompatible_fns.ld
@@ -193,7 +210,7 @@ LIB_SRC_C = $(addprefix lib/,\
 ifeq ($(MICROPY_FATFS), 1)
 LIB_SRC_C += \
 	lib/oofatfs/ff.c \
-	lib/oofatfs/option/unicode.c
+	lib/oofatfs/ffunicode.c
 endif
 
 DRIVERS_SRC_C = $(addprefix drivers/,\
@@ -213,6 +230,24 @@ SRC_QSTR += $(SRC_C) $(EXTMOD_SRC_C) $(LIB_SRC_C) $(DRIVERS_SRC_C)
 # Append any auto-generated sources that are needed by sources listed in SRC_QSTR
 SRC_QSTR_AUTO_DEPS +=
 
+################################################################################
+# Generate sdkconfig.h from sdkconfig
+
+$(SDKCONFIG_H): $(SDKCONFIG)
+	$(ECHO) "GEN $@"
+	$(Q)$(MKDIR) -p $(dir $@)
+	$(Q)$(PYTHON) $(ESPIDF)/tools/kconfig_new/confgen.py \
+		--output header $@ \
+		--config $< \
+		--kconfig $(ESPIDF)/Kconfig \
+		--env "IDF_TARGET=esp32" \
+		--env "IDF_CMAKE=n" \
+		--env "COMPONENT_KCONFIGS=$(ESPCOMP_KCONFIGS)" \
+		--env "COMPONENT_KCONFIGS_PROJBUILD=$(ESPCOMP_KCONFIGS_PROJBUILD)"
+	$(Q)touch $@
+
+$(HEADER_BUILD)/qstrdefs.generated.h: $(SDKCONFIG_H)
+
 ################################################################################
 # List of object files from the ESP32 IDF components
 
@@ -230,7 +265,17 @@ ESPIDF_DRIVER_O = $(addprefix $(ESPCOMP)/driver/,\
 $(BUILD)/$(ESPCOMP)/esp32/dport_access.o: CFLAGS += -Wno-array-bounds
 ESPIDF_ESP32_O = $(addprefix $(ESPCOMP)/esp32/,\
 	brownout.o \
+	cache_sram_mmu.o \
+	coexist.o \
+	dbg_stubs.o \
+	dport_panic_highint_hdl.o \
+	esp_err_to_name.o \
+	esp_himem.o \
 	panic.o \
+	pm_trace.o \
+	reset_reason.o \
+	restore.o \
+	stack_check.o \
 	esp_timer.o \
 	esp_timer_esp32.o \
 	ets_timer_legacy.o \
@@ -253,6 +298,8 @@ ESPIDF_ESP32_O = $(addprefix $(ESPCOMP)/esp32/,\
 	system_api.o \
 	hw_random.o \
 	phy_init.o \
+	pm_esp32.o \
+	pm_locks.o \
 	intr_alloc.o \
 	dport_access.o \
 	wifi_init.o \
@@ -262,6 +309,10 @@ ESPIDF_ESP32_O = $(addprefix $(ESPCOMP)/esp32/,\
 	spiram_psram.o \
 	)
 
+ESPIDF_ESP_RINGBUF_O = $(addprefix $(ESPCOMP)/esp_ringbuf/,\
+	ringbuf.o \
+	)
+
 ESPIDF_HEAP_O = $(addprefix $(ESPCOMP)/heap/,\
 	heap_caps.o \
 	heap_caps_init.o \
@@ -277,6 +328,7 @@ ESPIDF_SOC_O = $(addprefix $(ESPCOMP)/soc/,\
 	esp32/rtc_pm.o \
 	esp32/rtc_sleep.o \
 	esp32/rtc_time.o \
+	esp32/rtc_wdt.o \
 	esp32/soc_memory_layout.o \
 	esp32/spi_periph.o \
 	src/memory_layout_utils.o \
@@ -312,12 +364,13 @@ ESPIDF_PTHREAD_O = $(addprefix $(ESPCOMP)/pthread/,\
 # -Os because that generates subtly different code.
 # We also need custom CFLAGS for .c files because FreeRTOS has headers with
 # generic names (eg queue.h) which can clash with other files in the port.
-CFLAGS_ASM = -I$(ESPCOMP)/esp32/include -I$(ESPCOMP)/soc/esp32/include -I$(ESPCOMP)/freertos/include/freertos -I.
+CFLAGS_ASM = -I$(BUILD) -I$(ESPCOMP)/esp32/include -I$(ESPCOMP)/soc/esp32/include -I$(ESPCOMP)/freertos/include/freertos -I.
 $(BUILD)/$(ESPCOMP)/freertos/portasm.o: CFLAGS = $(CFLAGS_ASM)
 $(BUILD)/$(ESPCOMP)/freertos/xtensa_context.o: CFLAGS = $(CFLAGS_ASM)
 $(BUILD)/$(ESPCOMP)/freertos/xtensa_intr_asm.o: CFLAGS = $(CFLAGS_ASM)
 $(BUILD)/$(ESPCOMP)/freertos/xtensa_vectors.o: CFLAGS = $(CFLAGS_ASM)
-$(BUILD)/$(ESPCOMP)/freertos/%.o: CFLAGS = $(CFLAGS_BASE) -I. $(INC_ESPCOMP) -I$(ESPCOMP)/freertos/include/freertos -D_ESP_FREERTOS_INTERNAL
+$(BUILD)/$(ESPCOMP)/freertos/xtensa_vector_defaults.o: CFLAGS = $(CFLAGS_ASM)
+$(BUILD)/$(ESPCOMP)/freertos/%.o: CFLAGS = $(CFLAGS_BASE) -I. -I$(BUILD) $(INC_ESPCOMP) -I$(ESPCOMP)/freertos/include/freertos -D_ESP_FREERTOS_INTERNAL
 ESPIDF_FREERTOS_O = $(addprefix $(ESPCOMP)/freertos/,\
 	croutine.o \
 	event_groups.o \
@@ -326,7 +379,6 @@ ESPIDF_FREERTOS_O = $(addprefix $(ESPCOMP)/freertos/,\
 	portasm.o \
 	port.o \
 	queue.o \
-	ringbuf.o \
 	tasks.o \
 	timers.o \
 	xtensa_context.o \
@@ -366,6 +418,7 @@ ESPIDF_APP_TRACE_O = $(addprefix $(ESPCOMP)/app_trace/,\
 	)
 
 ESPIDF_APP_UPDATE_O = $(addprefix $(ESPCOMP)/app_update/,\
+	esp_app_desc.o \
 	esp_ota_ops.o \
 	)
 
@@ -404,6 +457,7 @@ ESPIDF_NGHTTP_O = $(addprefix $(ESPCOMP)/nghttp/,\
 	)
 
 ESPIDF_NVS_FLASH_O = $(addprefix $(ESPCOMP)/nvs_flash/,\
+	src/nvs_ops.o \
 	src/nvs_types.o \
 	src/nvs_page.o \
 	src/nvs_item_hash_list.o \
@@ -433,69 +487,72 @@ ESPIDF_ULP_O = $(addprefix $(ESPCOMP)/ulp/,\
 
 $(BUILD)/$(ESPCOMP)/lwip/%.o: CFLAGS += -Wno-address -Wno-unused-variable -Wno-unused-but-set-variable
 ESPIDF_LWIP_O = $(addprefix $(ESPCOMP)/lwip/,\
-	api/pppapi.o \
-	api/netbuf.o \
-	api/api_lib.o \
-	api/netifapi.o \
-	api/tcpip.o \
-	api/netdb.o \
-	api/err.o \
-	api/api_msg.o \
-	api/sockets.o \
-	apps/sntp/sntp.o \
-	apps/dhcpserver.o \
-	core/ipv4/ip_frag.o \
-	core/ipv4/dhcp.o \
-	core/ipv4/ip4_addr.o \
-	core/ipv4/igmp.o \
-	core/ipv4/ip4.o \
-	core/ipv4/autoip.o \
-	core/ipv4/icmp.o \
-	core/ipv6/ip6_frag.o \
-	core/ipv6/dhcp6.o \
-	core/ipv6/inet6.o \
-	core/ipv6/ip6_addr.o \
-	core/ipv6/ip6.o \
-	core/ipv6/nd6.o \
-	core/ipv6/mld6.o \
-	core/ipv6/ethip6.o \
-	core/ipv6/icmp6.o \
-	core/mem.o \
-	core/init.o \
-	core/memp.o \
-	core/sys.o \
-	core/tcp_in.o \
-	core/dns.o \
-	core/ip.o \
-	core/pbuf.o \
-	core/raw.o \
-	core/tcp.o \
-	core/def.o \
-	core/netif.o \
-	core/stats.o \
-	core/timers.o \
-	core/inet_chksum.o \
-	core/udp.o \
-	core/tcp_out.o \
-	netif/slipif.o \
-	netif/etharp.o \
-	netif/ethernet.o \
-	netif/lowpan6.o \
-	netif/ethernetif.o \
-	netif/ppp/ppp.o \
-	netif/ppp/magic.o \
-	netif/ppp/lcp.o \
-	netif/ppp/ipcp.o \
-	netif/ppp/auth.o \
-	netif/ppp/fsm.o \
-	netif/ppp/ipv6cp.o \
-	netif/ppp/utils.o \
-	netif/ppp/vj.o \
-	netif/ppp/pppos.o \
-	port/freertos/sys_arch.o \
-	port/netif/wlanif.o \
-	port/netif/ethernetif.o \
-	port/vfs_lwip.o \
+	apps/dhcpserver/dhcpserver.o \
+	lwip/src/api/netbuf.o \
+	lwip/src/api/api_lib.o \
+	lwip/src/api/netifapi.o \
+	lwip/src/api/tcpip.o \
+	lwip/src/api/netdb.o \
+	lwip/src/api/err.o \
+	lwip/src/api/api_msg.o \
+	lwip/src/api/sockets.o \
+	lwip/src/apps/sntp/sntp.o \
+	lwip/src/core/ipv4/dhcp.o \
+	lwip/src/core/ipv4/etharp.o \
+	lwip/src/core/ipv4/ip4_addr.o \
+	lwip/src/core/ipv4/igmp.o \
+	lwip/src/core/ipv4/ip4.o \
+	lwip/src/core/ipv4/autoip.o \
+	lwip/src/core/ipv4/icmp.o \
+	lwip/src/core/ipv6/ip6_frag.o \
+	lwip/src/core/ipv6/dhcp6.o \
+	lwip/src/core/ipv6/inet6.o \
+	lwip/src/core/ipv6/ip6_addr.o \
+	lwip/src/core/ipv6/ip6.o \
+	lwip/src/core/ipv6/nd6.o \
+	lwip/src/core/ipv6/mld6.o \
+	lwip/src/core/ipv6/ethip6.o \
+	lwip/src/core/ipv6/icmp6.o \
+	lwip/src/core/mem.o \
+	lwip/src/core/init.o \
+	lwip/src/core/memp.o \
+	lwip/src/core/sys.o \
+	lwip/src/core/tcp_in.o \
+	lwip/src/core/timeouts.o \
+	lwip/src/core/dns.o \
+	lwip/src/core/ip.o \
+	lwip/src/core/pbuf.o \
+	lwip/src/core/raw.o \
+	lwip/src/core/tcp.o \
+	lwip/src/core/def.o \
+	lwip/src/core/netif.o \
+	lwip/src/core/stats.o \
+	lwip/src/core/inet_chksum.o \
+	lwip/src/core/udp.o \
+	lwip/src/core/tcp_out.o \
+	lwip/src/netif/slipif.o \
+	lwip/src/netif/ethernet.o \
+	lwip/src/netif/lowpan6.o \
+	lwip/src/netif/ethernetif.o \
+	lwip/src/netif/ppp/auth.o \
+	lwip/src/netif/ppp/chap-md5.o \
+	lwip/src/netif/ppp/chap-new.o \
+	lwip/src/netif/ppp/fsm.o \
+	lwip/src/netif/ppp/ipcp.o \
+	lwip/src/netif/ppp/ipv6cp.o \
+	lwip/src/netif/ppp/lcp.o \
+	lwip/src/netif/ppp/magic.o \
+	lwip/src/netif/ppp/polarssl/md5.o \
+	lwip/src/netif/ppp/ppp.o \
+	lwip/src/netif/ppp/pppapi.o \
+	lwip/src/netif/ppp/pppos.o \
+	lwip/src/netif/ppp/upap.o \
+	lwip/src/netif/ppp/utils.o \
+	lwip/src/netif/ppp/vj.o \
+	port/esp32/freertos/sys_arch.o \
+	port/esp32/netif/ethernetif.o \
+	port/esp32/netif/wlanif.o \
+	port/esp32/vfs_lwip.o \
 	)
 
 ESPIDF_MBEDTLS_O = $(addprefix $(ESPCOMP)/mbedtls/,\
@@ -572,6 +629,7 @@ ESPIDF_MBEDTLS_O = $(addprefix $(ESPCOMP)/mbedtls/,\
 	mbedtls/library/pk_wrap.o \
 	port/esp_bignum.o \
 	port/esp_hardware.o \
+	port/esp_mem.o \
 	port/esp_sha1.o \
 	port/esp_sha256.o \
 	port/esp_sha512.o \
@@ -611,6 +669,7 @@ OBJ_ESPIDF =
 OBJ_ESPIDF += $(addprefix $(BUILD)/, $(ESPIDF_NEWLIB_O))
 OBJ_ESPIDF += $(addprefix $(BUILD)/, $(ESPIDF_DRIVER_O))
 OBJ_ESPIDF += $(addprefix $(BUILD)/, $(ESPIDF_ESP32_O))
+OBJ_ESPIDF += $(addprefix $(BUILD)/, $(ESPIDF_ESP_RINGBUF_O))
 OBJ_ESPIDF += $(addprefix $(BUILD)/, $(ESPIDF_HEAP_O))
 OBJ_ESPIDF += $(addprefix $(BUILD)/, $(ESPIDF_SOC_O))
 OBJ_ESPIDF += $(addprefix $(BUILD)/, $(ESPIDF_CXX_O))
@@ -634,6 +693,111 @@ OBJ_ESPIDF += $(addprefix $(BUILD)/, $(ESPIDF_SMARTCONFIG_ACK_O))
 OBJ_ESPIDF += $(addprefix $(BUILD)/, $(ESPIDF_SPI_FLASH_O))
 OBJ_ESPIDF += $(addprefix $(BUILD)/, $(ESPIDF_ULP_O))
 OBJ_ESPIDF += $(addprefix $(BUILD)/, $(ESPIDF_WPA_SUPPLICANT_O))
+
+$(OBJ_ESPIDF): $(SDKCONFIG_H)
+
+LIB_ESPIDF =
+LIB_ESPIDF += driver
+LIB_ESPIDF += esp32
+LIB_ESPIDF += esp_ringbuf
+LIB_ESPIDF += heap
+LIB_ESPIDF += soc
+LIB_ESPIDF += cxx
+LIB_ESPIDF += ethernet
+LIB_ESPIDF += expat
+LIB_ESPIDF += pthread
+LIB_ESPIDF += freertos
+LIB_ESPIDF += vfs
+LIB_ESPIDF += json
+LIB_ESPIDF += log
+LIB_ESPIDF += xtensa-debug-module
+LIB_ESPIDF += tcpip_adapter
+LIB_ESPIDF += app_trace
+LIB_ESPIDF += app_update
+LIB_ESPIDF += newlib
+LIB_ESPIDF += nghttp
+LIB_ESPIDF += nvs_flash
+LIB_ESPIDF += smartconfig_ack
+LIB_ESPIDF += spi_flash
+LIB_ESPIDF += ulp
+LIB_ESPIDF += lwip
+LIB_ESPIDF += mbedtls
+LIB_ESPIDF += wpa_supplicant
+
+BUILD_ESPIDF_LIB = $(BUILD)/esp-idf
+
+OBJ_ESPIDF_DIRS = $(sort $(dir $(OBJ_ESPIDF))) $(BUILD_ESPIDF_LIB) $(addprefix $(BUILD_ESPIDF_LIB)/,$(LIB_ESPIDF))
+$(OBJ_ESPIDF): | $(OBJ_ESPIDF_DIRS)
+$(OBJ_ESPIDF_DIRS):
+	$(MKDIR) -p $@
+
+define gen_espidf_lib_rule
+$(BUILD_ESPIDF_LIB)/$(1)/lib$(1).a: $(addprefix $$(BUILD)/,$(2))
+	$(ECHO) "AR $$@"
+	$(Q)$(AR) cru $$@ $$^
+endef
+
+$(eval $(call gen_espidf_lib_rule,driver,$(ESPIDF_DRIVER_O)))
+$(eval $(call gen_espidf_lib_rule,esp32,$(ESPIDF_ESP32_O)))
+$(eval $(call gen_espidf_lib_rule,esp_ringbuf,$(ESPIDF_ESP_RINGBUF_O)))
+$(eval $(call gen_espidf_lib_rule,heap,$(ESPIDF_HEAP_O)))
+$(eval $(call gen_espidf_lib_rule,soc,$(ESPIDF_SOC_O)))
+$(eval $(call gen_espidf_lib_rule,cxx,$(ESPIDF_CXX_O)))
+$(eval $(call gen_espidf_lib_rule,ethernet,$(ESPIDF_ETHERNET_O)))
+$(eval $(call gen_espidf_lib_rule,expat,$(ESPIDF_EXPAT_O)))
+$(eval $(call gen_espidf_lib_rule,pthread,$(ESPIDF_PTHREAD_O)))
+$(eval $(call gen_espidf_lib_rule,freertos,$(ESPIDF_FREERTOS_O)))
+$(eval $(call gen_espidf_lib_rule,vfs,$(ESPIDF_VFS_O)))
+$(eval $(call gen_espidf_lib_rule,json,$(ESPIDF_JSON_O)))
+$(eval $(call gen_espidf_lib_rule,log,$(ESPIDF_LOG_O)))
+$(eval $(call gen_espidf_lib_rule,xtensa-debug-module,$(ESPIDF_XTENSA_DEBUG_MODULE_O)))
+$(eval $(call gen_espidf_lib_rule,tcpip_adapter,$(ESPIDF_TCPIP_ADAPTER_O)))
+$(eval $(call gen_espidf_lib_rule,app_trace,$(ESPIDF_APP_TRACE_O)))
+$(eval $(call gen_espidf_lib_rule,app_update,$(ESPIDF_APP_UPDATE_O)))
+$(eval $(call gen_espidf_lib_rule,newlib,$(ESPIDF_NEWLIB_O)))
+$(eval $(call gen_espidf_lib_rule,nghttp,$(ESPIDF_NGHTTP_O)))
+$(eval $(call gen_espidf_lib_rule,nvs_flash,$(ESPIDF_NVS_FLASH_O)))
+$(eval $(call gen_espidf_lib_rule,smartconfig_ack,$(ESPIDF_SMARTCONFIG_ACK_O)))
+$(eval $(call gen_espidf_lib_rule,spi_flash,$(ESPIDF_SPI_FLASH_O)))
+$(eval $(call gen_espidf_lib_rule,ulp,$(ESPIDF_ULP_O)))
+$(eval $(call gen_espidf_lib_rule,lwip,$(ESPIDF_LWIP_O)))
+$(eval $(call gen_espidf_lib_rule,mbedtls,$(ESPIDF_MBEDTLS_O)))
+$(eval $(call gen_espidf_lib_rule,wpa_supplicant,$(ESPIDF_WPA_SUPPLICANT_O)))
+
+LIB = $(foreach lib,$(LIB_ESPIDF),$(BUILD_ESPIDF_LIB)/$(lib)/lib$(lib).a)
+
+################################################################################
+# ESP IDF ldgen
+
+LDGEN_FRAGMENTS = $(shell find $(ESPCOMP) -name "*.lf")
+LDGEN_SECTIONS_INFO = $(foreach lib,$(LIB_ESPIDF),$(BUILD_ESPIDF_LIB)/$(lib)/lib$(lib).a.sections_info)
+LDGEN_SECTION_INFOS = $(BUILD_ESPIDF_LIB)/ldgen.section_infos
+
+define gen_sections_info_rule
+$(1).sections_info: $(1)
+	$(ECHO) "GEN $(1).sections_info"
+	$(Q)$(OBJDUMP) -h $(1) > $(1).sections_info
+endef
+
+$(eval $(foreach lib,$(LIB_ESPIDF),$(eval $(call gen_sections_info_rule,$(BUILD_ESPIDF_LIB)/$(lib)/lib$(lib).a))))
+
+$(LDGEN_SECTION_INFOS): $(LDGEN_SECTIONS_INFO) $(ESPIDF)/make/ldgen.mk
+	$(Q)printf "$(foreach info,$(LDGEN_SECTIONS_INFO),$(info)\n)" > $@
+
+$(BUILD)/esp32.common.ld: $(ESPCOMP)/esp32/ld/esp32.common.ld.in $(LDGEN_FRAGMENTS) $(SDKCONFIG) $(LDGEN_SECTION_INFOS)
+	$(ECHO) "GEN $@"
+	$(Q)$(PYTHON) $(ESPIDF)/tools/ldgen/ldgen.py \
+		--input $< \
+		--output $@ \
+		--config $(SDKCONFIG) \
+		--kconfig $(ESPIDF)/Kconfig \
+		--fragments $(LDGEN_FRAGMENTS) \
+		--sections $(LDGEN_SECTION_INFOS) \
+		--env "IDF_TARGET=esp32" \
+		--env "IDF_CMAKE=n" \
+		--env "COMPONENT_KCONFIGS=$(ESPCOMP_KCONFIGS)" \
+		--env "COMPONENT_KCONFIGS_PROJBUILD=$(ESPCOMP_KCONFIGS_PROJBUILD)"
+
 ################################################################################
 # Main targets
 
@@ -659,7 +823,7 @@ erase:
 ################################################################################
 # Declarations to build the application
 
-OBJ = $(OBJ_MP) $(OBJ_ESPIDF)
+OBJ = $(OBJ_MP)
 
 APP_LD_ARGS =
 APP_LD_ARGS += $(LDFLAGS_MOD)
@@ -670,16 +834,17 @@ APP_LD_ARGS += $(LIBC_LIBM)
 APP_LD_ARGS += $(ESPCOMP)/esp32/libhal.a
 APP_LD_ARGS += -L$(ESPCOMP)/esp32/lib -lcore -lmesh -lnet80211 -lphy -lrtc -lpp -lwpa -lsmartconfig -lcoexist -lwps -lwpa2
 APP_LD_ARGS += $(OBJ)
+APP_LD_ARGS += $(LIB)
 APP_LD_ARGS += --end-group
 
-$(BUILD)/esp32_out.ld: sdkconfig.h
-	$(Q)$(CC) -I. -C -P -x c -E $(ESPCOMP)/esp32/ld/esp32.ld -o $@
+$(BUILD)/esp32_out.ld: $(SDKCONFIG_H)
+	$(Q)$(CC) -I$(BUILD) -C -P -x c -E $(ESPCOMP)/esp32/ld/esp32.ld -o $@
 
 $(BUILD)/application.bin: $(BUILD)/application.elf
 	$(ECHO) "Create $@"
 	$(Q)$(ESPTOOL) --chip esp32 elf2image --flash_mode $(FLASH_MODE) --flash_freq $(FLASH_FREQ) --flash_size $(FLASH_SIZE) $<
 
-$(BUILD)/application.elf: $(OBJ) $(BUILD)/esp32_out.ld
+$(BUILD)/application.elf: $(OBJ) $(LIB) $(BUILD)/esp32_out.ld $(BUILD)/esp32.common.ld
 	$(ECHO) "LINK $@"
 	$(Q)$(LD) $(LDFLAGS) -o $@ $(APP_LD_ARGS)
 	$(Q)$(SIZE) $@
@@ -757,13 +922,38 @@ $(BOOTLOADER_LIB_DIR)/libmicro-ecc.a: $(BOOTLOADER_LIB_MICRO_ECC_OBJ)
 	$(ECHO) "AR $@"
 	$(Q)$(AR) cr $@ $^
 
-# remaining object files
-BOOTLOADER_OBJ = $(addprefix $(BUILD)/bootloader/$(ESPCOMP)/,\
-	soc/esp32/rtc_clk.o \
-	soc/esp32/rtc_time.o \
-	soc/esp32/cpu_util.o \
+# libsoc.a
+$(BUILD)/bootloader/$(ESPCOMP)/soc/esp32/rtc_clk.o: CFLAGS += -fno-jump-tables -fno-tree-switch-conversion
+BOOTLOADER_LIB_ALL += soc
+BOOTLOADER_LIB_SOC_OBJ = $(addprefix $(BUILD)/bootloader/$(ESPCOMP)/soc/,\
+	esp32/cpu_util.o \
+	esp32/gpio_periph.o \
+	esp32/rtc_clk.o \
+	esp32/rtc_clk_init.o \
+	esp32/rtc_init.o \
+	esp32/rtc_periph.o \
+	esp32/rtc_pm.o \
+	esp32/rtc_sleep.o \
+	esp32/rtc_time.o \
+	esp32/rtc_wdt.o \
+	esp32/sdio_slave_periph.o \
+	esp32/sdmmc_periph.o \
+	esp32/soc_memory_layout.o \
+	esp32/spi_periph.o \
+	src/memory_layout_utils.o \
+	)
+$(BOOTLOADER_LIB_DIR)/libsoc.a: $(BOOTLOADER_LIB_SOC_OBJ)
+	$(ECHO) "AR $@"
+	$(Q)$(AR) cr $@ $^
+
+# libmain.a
+BOOTLOADER_LIB_ALL += main
+BOOTLOADER_LIB_MAIN_OBJ = $(addprefix $(BUILD)/bootloader/$(ESPCOMP)/,\
 	bootloader/subproject/main/bootloader_start.o \
 	)
+$(BOOTLOADER_LIB_DIR)/libmain.a: $(BOOTLOADER_LIB_MAIN_OBJ)
+	$(ECHO) "AR $@"
+	$(Q)$(AR) cr $@ $^
 
 # all objects files
 BOOTLOADER_OBJ_ALL = \
@@ -771,7 +961,10 @@ BOOTLOADER_OBJ_ALL = \
 	$(BOOTLOADER_LIB_LOG_OBJ) \
 	$(BOOTLOADER_LIB_SPI_FLASH_OBJ) \
 	$(BOOTLOADER_LIB_MICRO_ECC_OBJ) \
-	$(BOOTLOADER_OBJ)
+	$(BOOTLOADER_LIB_SOC_OBJ) \
+	$(BOOTLOADER_LIB_MAIN_OBJ)
+
+$(BOOTLOADER_OBJ_ALL): $(SDKCONFIG_H)
 
 BOOTLOADER_LIBS =
 BOOTLOADER_LIBS += -Wl,--start-group

ports/esp32/README.md

@@ -48,12 +48,12 @@ If you use WSL then follow the
 [Linux guidelines](https://esp-idf.readthedocs.io/en/latest/get-started/linux-setup.html)
 for the ESP-IDF instead of the Windows ones.
 
-The Espressif ESP-IDF instructions above only install pyserial for Python 2,
-so if you're running Python 3 or a non-system Python you'll also need to
-install `pyserial` (or `esptool`) so that the Makefile can flash the board
-and set parameters:
+You will also need either Python 2 or Python 3, along with the `pyserial` and
+`pyparsing` packages installed for the version of Python that you will be using
+(when building you can use, eg, `make PYTHON=python2` to specify the version
+used).  To install the required packages do:
 ```bash
-$ pip install pyserial
+$ pip install pyserial pyparsing
 ```
 
 Once everything is set up you should have a functioning toolchain with
@@ -78,7 +78,7 @@ ESPIDF = <path to root of esp-idf repository>
 #FLASH_MODE = qio
 #FLASH_SIZE = 4MB
 #CROSS_COMPILE = xtensa-esp32-elf-
-#CONFIG_SPIRAM_SUPPORT = 1
+#SDKCONFIG = boards/sdkconfig.spiram
 
 include Makefile
 ```
@@ -92,6 +92,17 @@ are `PORT` for the serial port of your esp32 module, and `FLASH_MODE`
 (which may need to be `dio` for some modules)
 and `FLASH_SIZE`.  See the Makefile for further information.
 
+The default ESP IDF configuration settings are provided in the file
+`boards/sdkconfig`, and this file is specified in the build by the make
+variable `SDKCONFIG`.  To use a custom configuration either set `SDKCONFIG`
+in your custom `makefile` (or `GNUmakefile`) or set this variable on the
+command line:
+```bash
+$ make SDKCONFIG=sdkconfig.myboard
+```
+The file `boards/sdkconfig.spiram` is provided for ESP32 modules that have
+external SPIRAM.
+
 Building the firmware
 ---------------------
 

ports/esp32/boards/sdkconfig

@@ -0,0 +1,30 @@
+# MicroPython on ESP32, ESP IDF configuration
+# The following options override the defaults
+
+CONFIG_IDF_TARGET="esp32"
+
+# Application manager
+CONFIG_APP_EXCLUDE_PROJECT_VER_VAR=y
+CONFIG_APP_EXCLUDE_PROJECT_NAME_VAR=y
+
+# Bootloader config
+CONFIG_LOG_BOOTLOADER_LEVEL_WARN=y
+
+# ESP32-specific
+CONFIG_ESP32_DEFAULT_CPU_FREQ_240=y
+CONFIG_TASK_WDT_CHECK_IDLE_TASK_CPU0=n
+CONFIG_TASK_WDT_CHECK_IDLE_TASK_CPU1=n
+CONFIG_ESP32_XTAL_FREQ_AUTO=y
+
+# Power Management
+CONFIG_PM_ENABLE=y
+
+# FreeRTOS
+CONFIG_FREERTOS_THREAD_LOCAL_STORAGE_POINTERS=2
+CONFIG_SUPPORT_STATIC_ALLOCATION=y
+CONFIG_ENABLE_STATIC_TASK_CLEAN_UP_HOOK=y
+
+# UDP
+CONFIG_PPP_SUPPORT=y
+CONFIG_PPP_PAP_SUPPORT=y
+CONFIG_PPP_CHAP_SUPPORT=y

ports/esp32/boards/sdkconfig.spiram

@@ -0,0 +1,33 @@
+# MicroPython on ESP32, ESP IDF configuration with SPIRAM support
+# The following options override the defaults
+
+CONFIG_IDF_TARGET="esp32"
+
+# Application manager
+CONFIG_APP_EXCLUDE_PROJECT_VER_VAR=y
+CONFIG_APP_EXCLUDE_PROJECT_NAME_VAR=y
+
+# Bootloader config
+CONFIG_LOG_BOOTLOADER_LEVEL_WARN=y
+
+# ESP32-specific
+CONFIG_ESP32_DEFAULT_CPU_FREQ_240=y
+CONFIG_SPIRAM_SUPPORT=y
+CONFIG_SPIRAM_IGNORE_NOTFOUND=y
+CONFIG_SPIRAM_USE_MEMMAP=y
+CONFIG_TASK_WDT_CHECK_IDLE_TASK_CPU0=n
+CONFIG_TASK_WDT_CHECK_IDLE_TASK_CPU1=n
+CONFIG_ESP32_XTAL_FREQ_AUTO=y
+
+# Power Management
+CONFIG_PM_ENABLE=y
+
+# FreeRTOS
+CONFIG_FREERTOS_THREAD_LOCAL_STORAGE_POINTERS=2
+CONFIG_SUPPORT_STATIC_ALLOCATION=y
+CONFIG_ENABLE_STATIC_TASK_CLEAN_UP_HOOK=y
+
+# UDP
+CONFIG_PPP_SUPPORT=y
+CONFIG_PPP_PAP_SUPPORT=y
+CONFIG_PPP_CHAP_SUPPORT=y

ports/esp32/esp32.custom_common.ld

@@ -1,274 +0,0 @@
-/*  Default entry point:  */
-ENTRY(call_start_cpu0);
-
-SECTIONS
-{
-  /* RTC fast memory holds RTC wake stub code,
-     including from any source file named rtc_wake_stub*.c
-  */
-  .rtc.text :
-  {
-    . = ALIGN(4);
-    *(.rtc.literal .rtc.text)
-    *rtc_wake_stub*.o(.literal .text .literal.* .text.*)
-  } > rtc_iram_seg
-
-  /* RTC slow memory holds RTC wake stub
-     data/rodata, including from any source file
-     named rtc_wake_stub*.c
-  */
-  .rtc.data :
-  {
-    _rtc_data_start = ABSOLUTE(.);
-    *(.rtc.data)
-    *(.rtc.rodata)
-    *rtc_wake_stub*.o(.data .rodata .data.* .rodata.* .bss .bss.*)
-    _rtc_data_end = ABSOLUTE(.);
-  } > rtc_slow_seg
-
-  /* RTC bss, from any source file named rtc_wake_stub*.c */
-  .rtc.bss (NOLOAD) :
-  {
-    _rtc_bss_start = ABSOLUTE(.);
-    *rtc_wake_stub*.o(.bss .bss.*)
-    *rtc_wake_stub*.o(COMMON)
-    _rtc_bss_end = ABSOLUTE(.);
-  } > rtc_slow_seg
-
-  /* This section holds data that should not be initialized at power up 
-     and will be retained during deep sleep. The section located in 
-     RTC SLOW Memory area. User data marked with RTC_NOINIT_ATTR will be placed
-     into this section. See the file "esp_attr.h" for more information.
-  */
-  .rtc_noinit (NOLOAD):
-  {
-    . = ALIGN(4);
-    _rtc_noinit_start = ABSOLUTE(.);
-    *(.rtc_noinit .rtc_noinit.*)
-    . = ALIGN(4) ;
-    _rtc_noinit_end = ABSOLUTE(.);
-  } > rtc_slow_seg
-
-  /* Send .iram0 code to iram */
-  .iram0.vectors :
-  {
-    _iram_start = ABSOLUTE(.);
-    /* Vectors go to IRAM */
-    _init_start = ABSOLUTE(.);
-    /* Vectors according to builds/RF-2015.2-win32/esp108_v1_2_s5_512int_2/config.html */
-    . = 0x0;
-    KEEP(*(.WindowVectors.text));
-    . = 0x180;
-    KEEP(*(.Level2InterruptVector.text));
-    . = 0x1c0;
-    KEEP(*(.Level3InterruptVector.text));
-    . = 0x200;
-    KEEP(*(.Level4InterruptVector.text));
-    . = 0x240;
-    KEEP(*(.Level5InterruptVector.text));
-    . = 0x280;
-    KEEP(*(.DebugExceptionVector.text));
-    . = 0x2c0;
-    KEEP(*(.NMIExceptionVector.text));
-    . = 0x300;
-    KEEP(*(.KernelExceptionVector.text));
-    . = 0x340;
-    KEEP(*(.UserExceptionVector.text));
-    . = 0x3C0;
-    KEEP(*(.DoubleExceptionVector.text));
-    . = 0x400;
-    *(.*Vector.literal)
-
-    *(.UserEnter.literal);
-    *(.UserEnter.text);
-    . = ALIGN (16);
-    *(.entry.text)
-    *(.init.literal)
-    *(.init)
-    _init_end = ABSOLUTE(.);
-  } > iram0_0_seg
-
-  .iram0.text :
-  {
-    /* Code marked as runnning out of IRAM */
-    _iram_text_start = ABSOLUTE(.);
-    *(.iram1 .iram1.*)
-    *freertos/*(.literal .text .literal.* .text.*)
-    *heap/multi_heap.o(.literal .text .literal.* .text.*)
-    *heap/multi_heap_poisoning.o(.literal .text .literal.* .text.*)
-    *esp32/panic.o(.literal .text .literal.* .text.*)
-    *esp32/core_dump.o(.literal .text .literal.* .text.*)
-    *app_trace/*(.literal .text .literal.* .text.*)
-    *xtensa-debug-module/eri.o(.literal .text .literal.* .text.*)
-    *librtc.a:(.literal .text .literal.* .text.*)
-    *soc/esp32/rtc_*.o(.literal .text .literal.* .text.*)
-    *soc/esp32/cpu_util.o(.literal .text .literal.* .text.*)
-    *libhal.a:(.literal .text .literal.* .text.*)
-    *libgcc.a:lib2funcs.o(.literal .text .literal.* .text.*)
-    *spi_flash/spi_flash_rom_patch.o(.literal .text .literal.* .text.*)
-    *libgcov.a:(.literal .text .literal.* .text.*)
-    INCLUDE esp32.spiram.rom-functions-iram.ld
-    *py/scheduler.o*(.literal .text .literal.* .text.*)
-    _iram_text_end = ABSOLUTE(.);
-    _iram_end = ABSOLUTE(.);
-  } > iram0_0_seg
-
-  .dram0.data :
-  {
-    _data_start = ABSOLUTE(.);
-    _bt_data_start = ABSOLUTE(.);
-    *libbt.a:(.data .data.*)
-    . = ALIGN (4);
-    _bt_data_end = ABSOLUTE(.);
-    _btdm_data_start = ABSOLUTE(.);
-    *libbtdm_app.a:(.data .data.*)
-    . = ALIGN (4);
-    _btdm_data_end = ABSOLUTE(.);
-    *(.data)
-    *(.data.*)
-    *(.gnu.linkonce.d.*)
-    *(.data1)
-    *(.sdata)
-    *(.sdata.*)
-    *(.gnu.linkonce.s.*)
-    *(.sdata2)
-    *(.sdata2.*)
-    *(.gnu.linkonce.s2.*)
-    *(.jcr)
-    *(.dram1 .dram1.*)
-    *esp32/panic.o(.rodata .rodata.*)
-    *libphy.a:(.rodata .rodata.*)
-    *soc/esp32/rtc_clk.o(.rodata .rodata.*)
-    *app_trace/app_trace.o(.rodata .rodata.*)
-    *libgcov.a:(.rodata .rodata.*)
-    *heap/multi_heap.o(.rodata .rodata.*)
-    *heap/multi_heap_poisoning.o(.rodata .rodata.*)
-    INCLUDE esp32.spiram.rom-functions-dram.ld
-    _data_end = ABSOLUTE(.);
-    . = ALIGN(4);
-  } > dram0_0_seg
-
-  /*This section holds data that should not be initialized at power up.
-    The section located in Internal SRAM memory region. The macro _NOINIT
-    can be used as attribute to place data into this section.
-    See the esp_attr.h file for more information.
-  */
-  .noinit (NOLOAD):
-  {
-    . = ALIGN(4);
-    _noinit_start = ABSOLUTE(.);
-    *(.noinit .noinit.*) 
-    . = ALIGN(4) ;
-    _noinit_end = ABSOLUTE(.);
-  } > dram0_0_seg
-
-  /* Shared RAM */
-  .dram0.bss (NOLOAD) :
-  {
-    . = ALIGN (8);
-    _bss_start = ABSOLUTE(.);
-    _bt_bss_start = ABSOLUTE(.);
-    *libbt.a:(.bss .bss.* COMMON)
-    . = ALIGN (4);
-    _bt_bss_end = ABSOLUTE(.);
-    _btdm_bss_start = ABSOLUTE(.);
-    *libbtdm_app.a:(.bss .bss.* COMMON)
-    . = ALIGN (4);
-    _btdm_bss_end = ABSOLUTE(.);
-    *(.dynsbss)
-    *(.sbss)
-    *(.sbss.*)
-    *(.gnu.linkonce.sb.*)
-    *(.scommon)
-    *(.sbss2)
-    *(.sbss2.*)
-    *(.gnu.linkonce.sb2.*)
-    *(.dynbss)
-    *(.bss)
-    *(.bss.*)
-    *(.share.mem)
-    *(.gnu.linkonce.b.*)
-    *(COMMON)
-    . = ALIGN (8);
-    _bss_end = ABSOLUTE(.);
-    /* The heap starts right after end of this section */
-    _heap_start = ABSOLUTE(.);
-  } > dram0_0_seg
-
-  .flash.rodata :
-  {
-    _rodata_start = ABSOLUTE(.);
-    *(.rodata)
-    *(.rodata.*)
-    *(.irom1.text) /* catch stray ICACHE_RODATA_ATTR */
-    *(.gnu.linkonce.r.*)
-    *(.rodata1)
-    __XT_EXCEPTION_TABLE_ = ABSOLUTE(.);
-    *(.xt_except_table)
-    *(.gcc_except_table .gcc_except_table.*)
-    *(.gnu.linkonce.e.*)
-    *(.gnu.version_r)
-    . = (. + 3) & ~ 3;
-    __eh_frame = ABSOLUTE(.);
-    KEEP(*(.eh_frame))
-    . = (. + 7) & ~ 3;
-    /*  C++ constructor and destructor tables, properly ordered:  */
-    __init_array_start = ABSOLUTE(.);
-    KEEP (*crtbegin.o(.ctors))
-    KEEP (*(EXCLUDE_FILE (*crtend.o) .ctors))
-    KEEP (*(SORT(.ctors.*)))
-    KEEP (*(.ctors))
-    __init_array_end = ABSOLUTE(.);
-    KEEP (*crtbegin.o(.dtors))
-    KEEP (*(EXCLUDE_FILE (*crtend.o) .dtors))
-    KEEP (*(SORT(.dtors.*)))
-    KEEP (*(.dtors))
-    /*  C++ exception handlers table:  */
-    __XT_EXCEPTION_DESCS_ = ABSOLUTE(.);
-    *(.xt_except_desc)
-    *(.gnu.linkonce.h.*)
-    __XT_EXCEPTION_DESCS_END__ = ABSOLUTE(.);
-    *(.xt_except_desc_end)
-    *(.dynamic)
-    *(.gnu.version_d)
-    /* Addresses of memory regions reserved via
-       SOC_RESERVE_MEMORY_REGION() */
-    soc_reserved_memory_region_start = ABSOLUTE(.);
-    KEEP (*(.reserved_memory_address))
-    soc_reserved_memory_region_end = ABSOLUTE(.);
-    _rodata_end = ABSOLUTE(.);
-    /* Literals are also RO data. */
-    _lit4_start = ABSOLUTE(.);
-    *(*.lit4)
-    *(.lit4.*)
-    *(.gnu.linkonce.lit4.*)
-    _lit4_end = ABSOLUTE(.);
-    . = ALIGN(4);
-    _thread_local_start = ABSOLUTE(.);
-    *(.tdata)
-    *(.tdata.*)
-    *(.tbss)
-    *(.tbss.*)
-    _thread_local_end = ABSOLUTE(.);
-    . = ALIGN(4);
-  } >drom0_0_seg
-
-  .flash.text :
-  {
-    _stext = .;
-    _text_start = ABSOLUTE(.);
-    *(.literal .text .literal.* .text.* .stub .gnu.warning .gnu.linkonce.literal.* .gnu.linkonce.t.*.literal .gnu.linkonce.t.*)
-    *(.irom0.text) /* catch stray ICACHE_RODATA_ATTR */
-    *(.fini.literal)
-    *(.fini)
-    *(.gnu.version)
-    _text_end = ABSOLUTE(.);
-    _etext = .;
-
-    /* Similar to _iram_start, this symbol goes here so it is
-       resolved by addr2line in preference to the first symbol in
-       the flash.text segment.
-    */
-    _flash_cache_start = ABSOLUTE(0);
-  } >iram0_2_seg
-}