all: Bump version to 1.15.

Signed-off-by: Damien George <damien@micropython.org>

.git-blame-ignore-revs

@@ -0,0 +1,32 @@
+# tests/run-tests.py: Reformat with Black.
+2a38d7103672580882fb621a5b76e8d26805d593
+
+# all: Update Python code to conform to latest black formatting.
+06659077a81b85882254cf0953c33b27614e018e
+
+# tools/uncrustify: Enable more opts to remove space between func and '('.
+77ed6f69ac35c1663a5633a8ee1d8a2446542204
+
+# tools/codeformat.py: Include extmod/{btstack,nimble} in code formatting.
+026fda605e03113d6e753290d65fed774418bc53
+
+# all: Format code to add space after C++-style comment start.
+84fa3312cfa7d2237d4b56952f2cd6e3591210c4
+
+# tests: Format all Python code with black, except tests in basics subdir.
+3dc324d3f1312e40d3a8ed87e7244966bb756f26
+
+# all: Remove spaces inside and around parenthesis.
+1a3e386c67e03a79eb768cb6e9f6777e002d6660
+
+# all: Remove spaces between nested paren and inside function arg paren.
+feb25775851ba0c04b8d1013716f442258879d9c
+
+# all: Reformat C and Python source code with tools/codeformat.py.
+69661f3343bedf86e514337cff63d96cc42f8859
+
+# stm32/usbdev: Convert files to unix line endings.
+abde0fa2267f9062b28c3c015d7662a550125cc6
+
+# all: Remove trailing spaces, per coding conventions.
+761e4c7ff62896c7d8f8c3dfc3cc98a4cc4f2f6f

.github/workflows/code_formatting.yml

@@ -0,0 +1,16 @@
+name: Check code formatting
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v1
+    - name: Install packages
+      run: source tools/ci.sh && ci_code_formatting_setup
+    - name: Run code formatting
+      run: source tools/ci.sh && ci_code_formatting_run
+    - name: Check code formatting
+      run: git diff --exit-code

.github/workflows/code_size.yml

@@ -0,0 +1,27 @@
+name: Check code size
+
+on:
+  push:
+  pull_request:
+    paths:
+      - '.github/workflows/*.yml'
+      - 'tools/**'
+      - 'py/**'
+      - 'extmod/**'
+      - 'lib/**'
+      - 'ports/bare-arm/**'
+      - 'ports/minimal/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 100
+    - name: Install packages
+      run: source tools/ci.sh && ci_code_size_setup
+    - name: Build
+      run: source tools/ci.sh && ci_code_size_build
+    - name: Compute code size difference
+      run: tools/metrics.py diff --error-threshold 0 ~/size0 ~/size1

.github/workflows/commit_formatting.yml

@@ -0,0 +1,14 @@
+name: Check commit message formatting
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: '100'
+    - uses: actions/setup-python@v1
+    - name: Check commit message formatting
+      run: source tools/ci.sh && ci_commit_formatting_run

.github/workflows/ports_cc3200.yml

@@ -0,0 +1,23 @@
+name: cc3200 port
+
+on:
+  push:
+  pull_request:
+    paths:
+      - '.github/workflows/*.yml'
+      - 'tools/**'
+      - 'py/**'
+      - 'extmod/**'
+      - 'lib/**'
+      - 'drivers/**'
+      - 'ports/cc3200/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install packages
+      run: source tools/ci.sh && ci_cc3200_setup
+    - name: Build
+      run: source tools/ci.sh && ci_cc3200_build

.github/workflows/ports_esp32.yml

@@ -0,0 +1,32 @@
+name: esp32 port
+
+on:
+  push:
+  pull_request:
+    paths:
+      - '.github/workflows/*.yml'
+      - 'tools/**'
+      - 'py/**'
+      - 'extmod/**'
+      - 'lib/**'
+      - 'drivers/**'
+      - 'ports/esp32/**'
+
+jobs:
+  build_idf402:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install packages
+      run: source tools/ci.sh && ci_esp32_idf402_setup
+    - name: Build
+      run: source tools/ci.sh && ci_esp32_build
+
+  build_idf43:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install packages
+      run: source tools/ci.sh && ci_esp32_idf43_setup
+    - name: Build
+      run: source tools/ci.sh && ci_esp32_build

.github/workflows/ports_esp8266.yml

@@ -0,0 +1,23 @@
+name: esp8266 port
+
+on:
+  push:
+  pull_request:
+    paths:
+      - '.github/workflows/*.yml'
+      - 'tools/**'
+      - 'py/**'
+      - 'extmod/**'
+      - 'lib/**'
+      - 'drivers/**'
+      - 'ports/esp8266/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install packages
+      run: source tools/ci.sh && ci_esp8266_setup && ci_esp8266_path >> $GITHUB_PATH
+    - name: Build
+      run: source tools/ci.sh && ci_esp8266_build

.github/workflows/ports_nrf.yml

@@ -0,0 +1,23 @@
+name: nrf port
+
+on:
+  push:
+  pull_request:
+    paths:
+      - '.github/workflows/*.yml'
+      - 'tools/**'
+      - 'py/**'
+      - 'extmod/**'
+      - 'lib/**'
+      - 'drivers/**'
+      - 'ports/nrf/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install packages
+      run: source tools/ci.sh && ci_nrf_setup
+    - name: Build
+      run: source tools/ci.sh && ci_nrf_build

.github/workflows/ports_powerpc.yml

@@ -0,0 +1,23 @@
+name: powerpc port
+
+on:
+  push:
+  pull_request:
+    paths:
+      - '.github/workflows/*.yml'
+      - 'tools/**'
+      - 'py/**'
+      - 'extmod/**'
+      - 'lib/**'
+      - 'drivers/**'
+      - 'ports/powerpc/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install packages
+      run: source tools/ci.sh && ci_powerpc_setup
+    - name: Build
+      run: source tools/ci.sh && ci_powerpc_build

.github/workflows/ports_qemu-arm.yml

@@ -0,0 +1,27 @@
+name: qemu-arm port
+
+on:
+  push:
+  pull_request:
+    paths:
+      - '.github/workflows/*.yml'
+      - 'tools/**'
+      - 'py/**'
+      - 'extmod/**'
+      - 'lib/**'
+      - 'drivers/**'
+      - 'ports/qemu-arm/**'
+      - 'tests/**'
+
+jobs:
+  build_and_test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install packages
+      run: source tools/ci.sh && ci_qemu_arm_setup
+    - name: Build and run test suite
+      run: source tools/ci.sh && ci_qemu_arm_build
+    - name: Print failures
+      if: failure()
+      run: grep --text "FAIL" ports/qemu-arm/build/console.out

.github/workflows/ports_rp2.yml

@@ -0,0 +1,23 @@
+name: rp2 port
+
+on:
+  push:
+  pull_request:
+    paths:
+      - '.github/workflows/*.yml'
+      - 'tools/**'
+      - 'py/**'
+      - 'extmod/**'
+      - 'lib/**'
+      - 'drivers/**'
+      - 'ports/rp2/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install packages
+      run: source tools/ci.sh && ci_rp2_setup
+    - name: Build
+      run: source tools/ci.sh && ci_rp2_build

.github/workflows/ports_samd.yml

@@ -0,0 +1,23 @@
+name: samd port
+
+on:
+  push:
+  pull_request:
+    paths:
+      - '.github/workflows/*.yml'
+      - 'tools/**'
+      - 'py/**'
+      - 'extmod/**'
+      - 'lib/**'
+      - 'drivers/**'
+      - 'ports/samd/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install packages
+      run: source tools/ci.sh && ci_samd_setup
+    - name: Build
+      run: source tools/ci.sh && ci_samd_build

.github/workflows/ports_stm32.yml

@@ -0,0 +1,32 @@
+name: stm32 port
+
+on:
+  push:
+  pull_request:
+    paths:
+      - '.github/workflows/*.yml'
+      - 'tools/**'
+      - 'py/**'
+      - 'extmod/**'
+      - 'lib/**'
+      - 'drivers/**'
+      - 'ports/stm32/**'
+
+jobs:
+  build_pyb:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install packages
+      run: source tools/ci.sh && ci_stm32_setup
+    - name: Build
+      run: source tools/ci.sh && ci_stm32_pyb_build
+
+  build_nucleo:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install packages
+      run: source tools/ci.sh && ci_stm32_setup
+    - name: Build
+      run: source tools/ci.sh && ci_stm32_nucleo_build

.github/workflows/ports_teensy.yml

@@ -0,0 +1,23 @@
+name: teensy port
+
+on:
+  push:
+  pull_request:
+    paths:
+      - '.github/workflows/*.yml'
+      - 'tools/**'
+      - 'py/**'
+      - 'extmod/**'
+      - 'lib/**'
+      - 'drivers/**'
+      - 'ports/teensy/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install packages
+      run: source tools/ci.sh && ci_teensy_setup
+    - name: Build
+      run: source tools/ci.sh && ci_teensy_build

.github/workflows/ports_unix.yml

@@ -0,0 +1,188 @@
+name: unix port
+
+on:
+  push:
+  pull_request:
+    paths:
+      - '.github/workflows/*.yml'
+      - 'tools/**'
+      - 'py/**'
+      - 'extmod/**'
+      - 'lib/**'
+      - 'examples/**'
+      - 'ports/unix/**'
+      - 'tests/**'
+
+jobs:
+  minimal:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build
+      run: source tools/ci.sh && ci_unix_minimal_build
+    - name: Run main test suite
+      run: source tools/ci.sh && ci_unix_minimal_run_tests
+    - name: Print failures
+      if: failure()
+      run: tests/run-tests.py --print-failures
+
+  reproducible:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build with reproducible date
+      run: source tools/ci.sh && ci_unix_minimal_build
+      env:
+        SOURCE_DATE_EPOCH: 1234567890
+    - name: Check reproducible build date
+      run: echo | ports/unix/micropython-minimal -i | grep 'on 2009-02-13;'
+
+  standard:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build
+      run: source tools/ci.sh && ci_unix_standard_build
+    - name: Run main test suite
+      run: source tools/ci.sh && ci_unix_standard_run_tests
+    - name: Run performance benchmarks
+      run: source tools/ci.sh && ci_unix_standard_run_perfbench
+    - name: Print failures
+      if: failure()
+      run: tests/run-tests.py --print-failures
+
+  coverage:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install packages
+      run: source tools/ci.sh && ci_unix_coverage_setup
+    - name: Build
+      run: source tools/ci.sh && ci_unix_coverage_build
+    - name: Run main test suite
+      run: source tools/ci.sh && ci_unix_coverage_run_tests
+    - name: Build native mpy modules
+      run: source tools/ci.sh && ci_native_mpy_modules_build
+    - name: Test importing .mpy generated by mpy_ld.py
+      run: source tools/ci.sh && ci_unix_coverage_run_native_mpy_tests
+    - name: Run lcov coverage analysis
+      run: |
+        mkdir -p coverage
+        lcov --rc lcov_branch_coverage=1 --directory ports/unix/build-coverage --capture --output-file coverage/lcov.info.all
+        lcov --remove coverage/lcov.info.all '*/lib/*' '*/ports/unix/*' '*/utils/*' --output-file coverage/lcov.info
+    - name: Send to coveralls
+      uses: coverallsapp/github-action@master
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Print failures
+      if: failure()
+      run: tests/run-tests.py --print-failures
+
+  coverage_32bit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install packages
+      run: source tools/ci.sh && ci_unix_32bit_setup
+    - name: Build
+      run: source tools/ci.sh && ci_unix_coverage_32bit_build
+    - name: Run main test suite
+      run: source tools/ci.sh && ci_unix_coverage_32bit_run_tests
+    - name: Build native mpy modules
+      run: source tools/ci.sh && ci_native_mpy_modules_32bit_build
+    - name: Test importing .mpy generated by mpy_ld.py
+      run: source tools/ci.sh && ci_unix_coverage_32bit_run_native_mpy_tests
+    - name: Print failures
+      if: failure()
+      run: tests/run-tests.py --print-failures
+
+  nanbox:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install packages
+      run: source tools/ci.sh && ci_unix_32bit_setup
+    - name: Build
+      run: source tools/ci.sh && ci_unix_nanbox_build
+    - name: Run main test suite
+      run: source tools/ci.sh && ci_unix_nanbox_run_tests
+    - name: Print failures
+      if: failure()
+      run: tests/run-tests.py --print-failures
+
+  float:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build
+      run: source tools/ci.sh && ci_unix_float_build
+    - name: Run main test suite
+      run: source tools/ci.sh && ci_unix_float_run_tests
+    - name: Print failures
+      if: failure()
+      run: tests/run-tests.py --print-failures
+
+  stackless_clang:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install packages
+      run: source tools/ci.sh && ci_unix_clang_setup
+    - name: Build
+      run: source tools/ci.sh && ci_unix_stackless_clang_build
+    - name: Run main test suite
+      run: source tools/ci.sh && ci_unix_stackless_clang_run_tests
+    - name: Print failures
+      if: failure()
+      run: tests/run-tests.py --print-failures
+
+  float_clang:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install packages
+      run: source tools/ci.sh && ci_unix_clang_setup
+    - name: Build
+      run: source tools/ci.sh && ci_unix_float_clang_build
+    - name: Run main test suite
+      run: source tools/ci.sh && ci_unix_float_clang_run_tests
+    - name: Print failures
+      if: failure()
+      run: tests/run-tests.py --print-failures
+
+  settrace:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build
+      run: source tools/ci.sh && ci_unix_settrace_build
+    - name: Run main test suite
+      run: source tools/ci.sh && ci_unix_settrace_run_tests
+    - name: Print failures
+      if: failure()
+      run: tests/run-tests.py --print-failures
+
+  settrace_stackless:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build
+      run: source tools/ci.sh && ci_unix_settrace_stackless_build
+    - name: Run main test suite
+      run: source tools/ci.sh && ci_unix_settrace_stackless_run_tests
+    - name: Print failures
+      if: failure()
+      run: tests/run-tests.py --print-failures
+
+  macos:
+    runs-on: macos-11.0
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v1
+    - name: Build
+      run: source tools/ci.sh && ci_unix_macos_build
+    - name: Run tests
+      run: source tools/ci.sh && ci_unix_macos_run_tests
+    - name: Print failures
+      if: failure()
+      run: tests/run-tests.py --print-failures

.github/workflows/ports_windows.yml

@@ -0,0 +1,23 @@
+name: windows port
+
+on:
+  push:
+  pull_request:
+    paths:
+      - '.github/workflows/*.yml'
+      - 'tools/**'
+      - 'py/**'
+      - 'extmod/**'
+      - 'lib/**'
+      - 'ports/unix/**'
+      - 'ports/windows/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install packages
+      run: source tools/ci.sh && ci_windows_setup
+    - name: Build
+      run: source tools/ci.sh && ci_windows_build

.github/workflows/ports_zephyr.yml

@@ -0,0 +1,24 @@
+name: zephyr port
+
+on:
+  push:
+  pull_request:
+    paths:
+      - '.github/workflows/ports_zephyr.yml'
+      - 'tools/**'
+      - 'py/**'
+      - 'extmod/**'
+      - 'lib/**'
+      - 'ports/zephyr/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install packages
+      run: source tools/ci.sh && ci_zephyr_setup
+    - name: Install Zephyr
+      run: source tools/ci.sh && ci_zephyr_install
+    - name: Build
+      run: source tools/ci.sh && ci_zephyr_build

.gitmodules

@@ -36,3 +36,9 @@
 [submodule "lib/nxp_driver"]
 	path = lib/nxp_driver
 	url = https://github.com/hathach/nxp_driver.git
+[submodule "lib/libhydrogen"]
+	path = lib/libhydrogen
+	url = https://github.com/jedisct1/libhydrogen.git
+[submodule "lib/pico-sdk"]
+	path = lib/pico-sdk
+	url = https://github.com/raspberrypi/pico-sdk.git

.travis.yml

@@ -1,395 +0,0 @@
-# global options
-dist: xenial
-language:
-  - c
-compiler:
-  - gcc
-cache:
-  directories:
-    - "${HOME}/persist"
-env:
-  global:
-    - MAKEOPTS="-j4"
-git:
-  submodules: false
-
-# define the successive stages
-stages:
-  - name: test
-
-# define the jobs for the stages
-# approx order of the jobs has longest running first to optimise total time
-jobs:
-  include:
-    # check code formatting
-    - stage: test
-      os: linux
-      dist: bionic
-      name: "code formatting"
-      before_install:
-        - sudo apt-add-repository --yes --update ppa:pybricks/ppa
-      install:
-        - sudo apt-get install uncrustify python3-pip
-        - uncrustify --version
-        - pip3 install --user black
-        - black --version
-      script:
-        - tools/codeformat.py
-        - git diff --exit-code
-
-    # zephyr port
-    - stage: test
-      name: "zephyr port build"
-      services:
-        - docker
-      before_install:
-        - docker pull zephyrprojectrtos/ci:v0.11.8
-        - >
-          docker run --name zephyr-ci -d -it
-          -v "$(pwd)":/micropython
-          -e ZEPHYR_SDK_INSTALL_DIR=/opt/sdk/zephyr-sdk-0.11.3
-          -e ZEPHYR_TOOLCHAIN_VARIANT=zephyr
-          -w /micropython/ports/zephyr
-          zephyrprojectrtos/ci:v0.11.8
-        - docker ps -a
-      install:
-        - docker exec zephyr-ci west init --mr v2.3.0 /zephyrproject
-        - docker exec -w /zephyrproject zephyr-ci west update
-        - docker exec -w /zephyrproject zephyr-ci west zephyr-export
-      script:
-        - docker exec zephyr-ci bash -c "make clean; ./make-minimal ${MAKEOPTS}"
-        - docker exec zephyr-ci bash -c "make clean; ./make-minimal ${MAKEOPTS} BOARD=frdm_k64f"
-        - docker exec zephyr-ci bash -c "make clean; make ${MAKEOPTS}"
-        - docker exec zephyr-ci bash -c "make clean; make ${MAKEOPTS} BOARD=frdm_k64f"
-        - docker exec zephyr-ci bash -c "make clean; make ${MAKEOPTS} BOARD=mimxrt1050_evk"
-        - docker exec zephyr-ci bash -c "make clean; make ${MAKEOPTS} BOARD=reel_board"
-
-    # unix port on OSX (first in list because the build VM takes a long time to start)
-    - stage: test
-      os: osx
-      osx_image: xcode11.3
-      name: "unix port build with clang on OSX"
-      env:
-        - PKG_CONFIG_PATH=/usr/local/opt/libffi/lib/pkgconfig
-      script:
-        - make ${MAKEOPTS} -C mpy-cross
-        - make ${MAKEOPTS} -C ports/unix submodules
-        - make ${MAKEOPTS} -C ports/unix deplibs
-        - make ${MAKEOPTS} -C ports/unix
-        # OSX has poor time resolution and the following tests do not have the correct output
-        - (cd tests && ./run-tests --exclude 'uasyncio_(basic|heaplock|lock|wait_task)')
-        # check for additional compiler errors/warnings
-        - make ${MAKEOPTS} -C ports/unix VARIANT=coverage submodules
-        - make ${MAKEOPTS} -C ports/unix VARIANT=coverage
-      after_failure:
-        - tests/run-tests --print-failures
-
-    # stm32 port
-    - stage: test
-      name: "stm32 port build"
-      install:
-        # need newer gcc version for Cortex-M7 support
-        - sudo add-apt-repository -y ppa:team-gcc-arm-embedded/ppa
-        - sudo apt-get update -qq || true
-        - sudo apt-get install gcc-arm-embedded libnewlib-arm-none-eabi
-        - arm-none-eabi-gcc --version
-      script:
-        - make ${MAKEOPTS} -C mpy-cross
-        - make ${MAKEOPTS} -C ports/stm32 submodules
-        - git submodule update --init lib/btstack
-        - make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_F091RC
-        - make ${MAKEOPTS} -C ports/stm32 BOARD=PYBV11 MICROPY_PY_WIZNET5K=5200 MICROPY_PY_CC3K=1
-        - make ${MAKEOPTS} -C ports/stm32 BOARD=PYBD_SF2
-        - make ${MAKEOPTS} -C ports/stm32 BOARD=PYBD_SF6 NANBOX=1 MICROPY_BLUETOOTH_NIMBLE=0 MICROPY_BLUETOOTH_BTSTACK=1
-        - make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_H743ZI CFLAGS_EXTRA='-DMICROPY_PY_THREAD=1'
-        - make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_L073RZ
-        - make ${MAKEOPTS} -C ports/stm32 BOARD=STM32L476DISC
-        - make ${MAKEOPTS} -C ports/stm32 BOARD=NUCLEO_WB55
-        - make ${MAKEOPTS} -C ports/stm32/mboot BOARD=PYBV10 CFLAGS_EXTRA='-DMBOOT_FSLOAD=1 -DMBOOT_VFS_LFS2=1'
-        - make ${MAKEOPTS} -C ports/stm32/mboot BOARD=PYBD_SF6
-        - make ${MAKEOPTS} -C ports/stm32/mboot BOARD=NUCLEO_WB55
-
-    # qemu-arm port
-    - stage: test
-      dist: bionic # needed for more recent version of qemu-system-arm with mps2-an385 target
-      name: "qemu-arm port build and tests"
-      install:
-        - sudo apt-get install gcc-arm-none-eabi libnewlib-arm-none-eabi qemu-system
-        - arm-none-eabi-gcc --version
-        - qemu-system-arm --version
-      script:
-        - make ${MAKEOPTS} -C mpy-cross
-        - make ${MAKEOPTS} -C ports/qemu-arm CFLAGS_EXTRA=-DMP_ENDIANNESS_BIG=1
-        - make ${MAKEOPTS} -C ports/qemu-arm clean
-        - make ${MAKEOPTS} -C ports/qemu-arm -f Makefile.test test
-      after_failure:
-        - grep --text "FAIL" ports/qemu-arm/build/console.out
-
-    # unix coverage
-    - stage: test
-      name: "unix coverage build and tests"
-      install:
-        - sudo apt-get install python3-pip
-        - sudo pip install cpp-coveralls
-        - sudo pip3 install setuptools
-        - sudo pip3 install pyelftools
-        - gcc --version
-        - python3 --version
-      script:
-        - make ${MAKEOPTS} -C mpy-cross
-        - make ${MAKEOPTS} -C ports/unix VARIANT=coverage submodules
-        - make ${MAKEOPTS} -C ports/unix VARIANT=coverage deplibs
-        - make ${MAKEOPTS} -C ports/unix VARIANT=coverage
-        # run the main test suite
-        - make -C ports/unix VARIANT=coverage test_full
-        - (cd tests && MICROPY_CPYTHON3=python3 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-multitests.py multi_net/*.py) || travis_terminate 1
-        # test building native mpy modules
-        - make -C examples/natmod/features1 ARCH=x64
-        - make -C examples/natmod/features2 ARCH=x64
-        - make -C examples/natmod/btree ARCH=x64
-        - make -C examples/natmod/framebuf ARCH=x64
-        - make -C examples/natmod/uheapq ARCH=x64
-        - make -C examples/natmod/urandom ARCH=x64
-        - make -C examples/natmod/ure ARCH=x64
-        - make -C examples/natmod/uzlib ARCH=x64
-        # test importing .mpy generated by mpy_ld.py
-        - MICROPYPATH=examples/natmod/features2 ./ports/unix/micropython-coverage -m features2
-        - (cd tests && ./run-natmodtests.py extmod/{btree*,framebuf*,uheapq*,ure*,uzlib*}.py)
-        # run coveralls coverage analysis (try to, even if some builds/tests failed)
-        - (cd ports/unix && coveralls --root ../.. --build-root . --gcov $(which gcov) --gcov-options '\-o build-coverage/' --include py --include extmod)
-      after_failure:
-        - tests/run-tests --print-failures
-
-    # unix coverage 32-bit
-    - stage: test
-      name: "unix coverage 32-bit build and tests"
-      install:
-        - sudo apt-get install gcc-multilib libffi-dev:i386
-        - sudo apt-get install python3-pip
-        - sudo pip3 install setuptools
-        - sudo pip3 install pyelftools
-        - gcc --version
-        - python3 --version
-      script:
-        - make ${MAKEOPTS} -C mpy-cross
-        - make ${MAKEOPTS} -C ports/unix MICROPY_FORCE_32BIT=1 VARIANT=coverage submodules
-        - make ${MAKEOPTS} -C ports/unix MICROPY_FORCE_32BIT=1 VARIANT=coverage deplibs
-        - make ${MAKEOPTS} -C ports/unix MICROPY_FORCE_32BIT=1 VARIANT=coverage
-        # run the main test suite
-        - make -C ports/unix MICROPY_FORCE_32BIT=1 VARIANT=coverage test_full || travis_terminate 1
-        # test building native mpy modules
-        - make -C examples/natmod/features1 ARCH=x86
-        - make -C examples/natmod/features2 ARCH=x86
-        - make -C examples/natmod/btree ARCH=x86
-        - make -C examples/natmod/framebuf ARCH=x86
-        - make -C examples/natmod/uheapq ARCH=x86
-        - make -C examples/natmod/urandom ARCH=x86
-        - make -C examples/natmod/ure ARCH=x86
-        - make -C examples/natmod/uzlib ARCH=x86
-        # test importing .mpy generated by mpy_ld.py
-        - MICROPYPATH=examples/natmod/features2 ./ports/unix/micropython-coverage -m features2
-        - (cd tests && ./run-natmodtests.py --arch x86 extmod/{btree*,framebuf*,uheapq*,ure*,uzlib*}.py)
-      after_failure:
-        - tests/run-tests --print-failures
-
-    # standard unix port
-    - stage: test
-      name: "unix port build and tests"
-      script:
-        - make ${MAKEOPTS} -C mpy-cross
-        - make ${MAKEOPTS} -C ports/unix submodules
-        - make ${MAKEOPTS} -C ports/unix deplibs
-        - make ${MAKEOPTS} -C ports/unix
-        - make ${MAKEOPTS} -C ports/unix test
-        - (cd tests && MICROPY_CPYTHON3=python3 MICROPY_MICROPYTHON=../ports/unix/micropython ./run-perfbench.py 1000 1000)
-      after_failure:
-        - tests/run-tests --print-failures
-
-    # unix nanbox/float (and using Python 2 to check it can run the build scripts)
-    - stage: test
-      name: "unix nanbox/float port build and tests"
-      install:
-        - sudo apt-get install gcc-multilib libffi-dev:i386
-      script:
-        - make ${MAKEOPTS} -C mpy-cross PYTHON=python2
-        - make ${MAKEOPTS} -C ports/unix VARIANT=nanbox submodules
-        - make ${MAKEOPTS} -C ports/unix PYTHON=python2 VARIANT=nanbox deplibs
-        - make ${MAKEOPTS} -C ports/unix PYTHON=python2 VARIANT=nanbox
-        - make ${MAKEOPTS} -C ports/unix PYTHON=python2 VARIANT=nanbox test_full || travis_terminate 1
-        - make ${MAKEOPTS} -C ports/unix clean
-        - make ${MAKEOPTS} -C ports/unix CFLAGS_EXTRA="-DMICROPY_FLOAT_IMPL=MICROPY_FLOAT_IMPL_FLOAT"
-        - make ${MAKEOPTS} -C ports/unix test
-      after_failure:
-        - tests/run-tests --print-failures
-
-    # unix stackless/float with clang
-    - stage: test
-      name: "unix stackless/float port build and tests with clang"
-      install:
-        - sudo apt-get install clang
-      script:
-        - make ${MAKEOPTS} -C mpy-cross CC=clang
-        - make ${MAKEOPTS} -C ports/unix submodules
-        - make ${MAKEOPTS} -C ports/unix CC=clang CFLAGS_EXTRA="-DMICROPY_STACKLESS=1 -DMICROPY_STACKLESS_STRICT=1"
-        - make ${MAKEOPTS} -C ports/unix CC=clang test || travis_terminate 1
-        - make ${MAKEOPTS} -C ports/unix clean
-        - make ${MAKEOPTS} -C ports/unix CC=clang CFLAGS_EXTRA="-DMICROPY_FLOAT_IMPL=MICROPY_FLOAT_IMPL_FLOAT"
-        - make ${MAKEOPTS} -C ports/unix CC=clang test
-      after_failure:
-        - tests/run-tests --print-failures
-
-    # unix with sys.settrace
-    - stage: test
-      name: "unix port with sys.settrace build and tests"
-      script:
-        - make ${MAKEOPTS} -C mpy-cross
-        - make ${MAKEOPTS} -C ports/unix MICROPY_PY_BTREE=0 MICROPY_PY_FFI=0 MICROPY_PY_USSL=0 CFLAGS_EXTRA="-DMICROPY_PY_SYS_SETTRACE=1" test || travis_terminate 1
-        - make ${MAKEOPTS} -C ports/unix clean
-        - make ${MAKEOPTS} -C ports/unix MICROPY_PY_BTREE=0 MICROPY_PY_FFI=0 MICROPY_PY_USSL=0 CFLAGS_EXTRA="-DMICROPY_STACKLESS=1 -DMICROPY_STACKLESS_STRICT=1 -DMICROPY_PY_SYS_SETTRACE=1" test
-      after_failure:
-        - tests/run-tests --print-failures
-
-    # minimal unix port with tests
-    - stage: test
-      name: "minimal unix port build and tests"
-      script:
-        - make ${MAKEOPTS} -C ports/unix VARIANT=minimal
-        - (cd tests && MICROPY_CPYTHON3=python3 MICROPY_MICROPYTHON=../ports/unix/micropython-minimal ./run-tests -e exception_chain -e self_type_check -e subclass_native_init -d basics)
-      after_failure:
-        - tests/run-tests --print-failures
-
-    # windows port via mingw
-    - stage: test
-      name: "windows port build via mingw"
-      install:
-        - sudo apt-get install gcc-mingw-w64
-      script:
-        - make ${MAKEOPTS} -C mpy-cross
-        - make ${MAKEOPTS} -C ports/windows CROSS_COMPILE=i686-w64-mingw32-
-
-    # esp32 w/ESP-IDFv3 port
-    - stage: test
-      name: "esp32 ESP-IDFv3 port build"
-      install:
-        - sudo apt-get install python3-pip
-        - sudo pip3 install 'pyparsing<2.4'
-        - curl -L https://dl.espressif.com/dl/xtensa-esp32-elf-linux64-1.22.0-80-g6c4433a-5.2.0.tar.gz | tar zxf -
-        - export PATH=$(pwd)/xtensa-esp32-elf/bin:$PATH
-        - git clone https://github.com/espressif/esp-idf.git
-        - export IDF_PATH=$(pwd)/esp-idf
-      script:
-        - make ${MAKEOPTS} -C mpy-cross
-        - git -C esp-idf checkout $(grep "ESPIDF_SUPHASH_V3 :=" ports/esp32/Makefile | cut -d " " -f 3)
-        - git -C esp-idf submodule update --init components/json/cJSON components/esp32/lib components/esptool_py/esptool components/expat/expat components/lwip/lwip components/mbedtls/mbedtls components/micro-ecc/micro-ecc components/nghttp/nghttp2 components/nimble components/bt
-        - make ${MAKEOPTS} -C ports/esp32 submodules
-        - make ${MAKEOPTS} -C ports/esp32
-
-    # esp32 w/ESP-IDFv4 port
-    - stage: test
-      name: "esp32 ESP-IDFv4 port build"
-      install:
-        - sudo apt-get install python3-pip
-        - sudo pip3 install 'pyparsing<2.4'
-        - curl -L https://dl.espressif.com/dl/xtensa-esp32-elf-gcc8_2_0-esp-2019r2-linux-amd64.tar.gz | tar zxf -
-        - export PATH=$(pwd)/xtensa-esp32-elf/bin:$PATH
-        - git clone https://github.com/espressif/esp-idf.git
-        - export IDF_PATH=$(pwd)/esp-idf
-      script:
-        - make ${MAKEOPTS} -C mpy-cross
-        - git -C esp-idf checkout $(grep "ESPIDF_SUPHASH_V4 :=" ports/esp32/Makefile | cut -d " " -f 3)
-        - git -C esp-idf submodule update --init components/bt/controller/lib components/bt/host/nimble/nimble components/esp_wifi/lib_esp32 components/esptool_py/esptool components/lwip/lwip components/mbedtls/mbedtls
-        - make ${MAKEOPTS} -C ports/esp32 submodules
-        - make ${MAKEOPTS} -C ports/esp32
-
-    # esp8266 port
-    - stage: test
-      name: "esp8266 port build"
-      install:
-        - wget https://github.com/jepler/esp-open-sdk/releases/download/2018-06-10/xtensa-lx106-elf-standalone.tar.gz
-        - zcat xtensa-lx106-elf-standalone.tar.gz | tar x
-        - export PATH=$(pwd)/xtensa-lx106-elf/bin:$PATH
-      script:
-        - make ${MAKEOPTS} -C mpy-cross
-        - make ${MAKEOPTS} -C ports/esp8266 submodules
-        - make ${MAKEOPTS} -C ports/esp8266
-        - make ${MAKEOPTS} -C ports/esp8266 BOARD=GENERIC_512K
-        - make ${MAKEOPTS} -C ports/esp8266 BOARD=GENERIC_1M
-
-    # nrf port
-    - stage: test
-      name: "nrf port build"
-      install:
-        # need newer gcc version for Cortex-M33 support
-        - sudo add-apt-repository -y ppa:team-gcc-arm-embedded/ppa
-        - sudo apt-get update -qq || true
-        - sudo apt-get install gcc-arm-embedded
-        - sudo apt-get install libnewlib-arm-none-eabi
-        - arm-none-eabi-gcc --version
-      script:
-        - ports/nrf/drivers/bluetooth/download_ble_stack.sh s140_nrf52_6_1_1
-        - make ${MAKEOPTS} -C ports/nrf submodules
-        - make ${MAKEOPTS} -C ports/nrf BOARD=pca10040
-        - make ${MAKEOPTS} -C ports/nrf BOARD=microbit
-        - make ${MAKEOPTS} -C ports/nrf BOARD=pca10056 SD=s140
-        - make ${MAKEOPTS} -C ports/nrf BOARD=pca10090
-
-    # bare-arm and minimal ports, with size-diff check
-    - stage: test
-      name: "bare-arm and minimal ports build and size-diff check"
-      install:
-        - sudo apt-get install gcc-multilib libffi-dev:i386 gcc-arm-none-eabi libnewlib-arm-none-eabi
-        - gcc --version
-        - arm-none-eabi-gcc --version
-      script:
-        # starts off at either the ref/pull/N/merge FETCH_HEAD, or the current branch HEAD
-        - git checkout -b pull_request # save the current location
-        - git remote add upstream https://github.com/micropython/micropython.git
-        - git fetch --depth=100 upstream
-        # build reference, save to size0
-        # ignore any errors with this build, in case master is failing
-        - git checkout `git merge-base --fork-point upstream/master pull_request`
-        - git show -s
-        - tools/metrics.py clean bm
-        - tools/metrics.py build bm | tee ~/size0 || true
-        # build PR/branch, save to size1
-        - git checkout pull_request
-        - git log upstream/master..HEAD
-        - tools/metrics.py clean bm
-        - tools/metrics.py build bm | tee ~/size1 || travis_terminate 1
-        # compute diff of the code sizes
-        - tools/metrics.py diff --error-threshold 0 ~/size0 ~/size1
-
-    # cc3200 port
-    - stage: test
-      name: "cc3200 port build"
-      install:
-        - sudo apt-get install gcc-arm-none-eabi libnewlib-arm-none-eabi
-      script:
-        - make ${MAKEOPTS} -C ports/cc3200 BTARGET=application BTYPE=release
-        - make ${MAKEOPTS} -C ports/cc3200 BTARGET=bootloader  BTYPE=release
-
-    # samd port
-    - stage: test
-      name: "samd port build"
-      install:
-        - sudo apt-get install gcc-arm-none-eabi libnewlib-arm-none-eabi
-      script:
-        - make ${MAKEOPTS} -C ports/samd submodules
-        - make ${MAKEOPTS} -C ports/samd
-
-    # teensy port
-    - stage: test
-      name: "teensy port build"
-      install:
-        - sudo apt-get install gcc-arm-none-eabi libnewlib-arm-none-eabi
-      script:
-        - make ${MAKEOPTS} -C ports/teensy
-
-    # powerpc port
-    - stage: test
-      name: "powerpc port build"
-      install:
-        - sudo apt-get install gcc-powerpc64le-linux-gnu libc6-dev-ppc64el-cross
-      script:
-        - make ${MAKEOPTS} -C ports/powerpc UART=potato
-        - make ${MAKEOPTS} -C ports/powerpc UART=lpc_serial

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2013-2020 Damien P. George
+Copyright (c) 2013-2021 Damien P. George
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.com/micropython/micropython.png?branch=master)](https://travis-ci.com/micropython/micropython) [![Coverage Status](https://coveralls.io/repos/micropython/micropython/badge.png?branch=master)](https://coveralls.io/r/micropython/micropython?branch=master)
+[![CI badge](https://github.com/micropython/micropython/workflows/unix%20port/badge.svg)](https://github.com/micropython/micropython/actions?query=branch%3Amaster+event%3Apush) [![Coverage badge](https://coveralls.io/repos/micropython/micropython/badge.png?branch=master)](https://coveralls.io/r/micropython/micropython?branch=master)
 
 The MicroPython project
 =======================

docs/conf.py

@@ -66,7 +66,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'MicroPython'
-copyright = '2014-2020, Damien P. George, Paul Sokolovsky, and contributors'
+copyright = '2014-2021, Damien P. George, Paul Sokolovsky, and contributors'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -74,7 +74,7 @@ copyright = '2014-2020, Damien P. George, Paul Sokolovsky, and contributors'
 #
 # We don't follow "The short X.Y version" vs "The full version, including alpha/beta/rc tags"
 # breakdown, so use the same version identifier for both to avoid confusion.
-version = release = '1.13'
+version = release = '1.15'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/develop/cmodules.rst

@@ -8,7 +8,8 @@ limitations with the Python environment, often due to an inability to access
 certain hardware resources or Python speed limitations.
 
 If your limitations can't be resolved with suggestions in :ref:`speed_python`,
-writing some or all of your module in C is a viable option.
+writing some or all of your module in C (and/or C++ if implemented for your port)
+is a viable option.
 
 If your module is designed to access or work with commonly available
 hardware or libraries please consider implementing it inside the MicroPython
@@ -17,7 +18,11 @@ If however you're targeting obscure or proprietary systems it may make
 more sense to keep this external to the main MicroPython repository.
 
 This chapter describes how to compile such external modules into the
-MicroPython executable or firmware image.
+MicroPython executable or firmware image.  Both Make and CMake build
+tools are supported, and when writing an external module it's a good idea to
+add the build files for both of these tools so the module can be used on all
+ports.  But when compiling a particular port you will only need to use one
+method of building, either Make or CMake.
 
 An alternative approach is to use :ref:`natmod` which allows writing custom C
 code that is placed in a .mpy file, which can be imported dynamically in to
@@ -29,7 +34,7 @@ Structure of an external C module
 
 A MicroPython user C module is a directory with the following files:
 
-* ``*.c`` and/or ``*.h`` source code files for your module.
+* ``*.c`` / ``*.cpp`` / ``*.h`` source code files for your module.
 
   These will typically include the low level functionality being implemented and
   the MicroPython binding functions to expose the functions and module(s).
@@ -44,12 +49,37 @@ A MicroPython user C module is a directory with the following files:
   in your ``micropython.mk`` to a local make variable,
   eg ``EXAMPLE_MOD_DIR := $(USERMOD_DIR)``
 
-  Your ``micropython.mk`` must add your modules C files relative to your
+  Your ``micropython.mk`` must add your modules source files relative to your
   expanded copy of ``$(USERMOD_DIR)`` to ``SRC_USERMOD``, eg
   ``SRC_USERMOD += $(EXAMPLE_MOD_DIR)/example.c``
 
-  If you have custom ``CFLAGS`` settings or include folders to define, these
-  should be added to ``CFLAGS_USERMOD``.
+  If you have custom compiler options (like ``-I`` to add directories to search
+  for header files), these should be added to ``CFLAGS_USERMOD`` for C code
+  and to ``CXXFLAGS_USERMOD`` for C++ code.
+
+* ``micropython.cmake`` contains the CMake configuration for this module.
+
+  In ``micropython.cmake``, you may use ``${CMAKE_CURRENT_LIST_DIR}`` as the path to
+  the current module.
+
+  Your ``micropython.cmake`` should define an ``INTERFACE`` library and associate
+  your source files, compile definitions and include directories with it.
+  The library should then be linked to the ``usermod`` target.
+
+  .. code-block:: cmake
+
+      add_library(usermod_cexample INTERFACE)
+
+      target_sources(usermod_cexample INTERFACE
+          ${CMAKE_CURRENT_LIST_DIR}/examplemodule.c
+      )
+
+      target_include_directories(usermod_cexample INTERFACE
+          ${CMAKE_CURRENT_LIST_DIR}
+      )
+
+      target_link_libraries(usermod INTERFACE usermod_cexample)
+
 
   See below for full usage example.
 
@@ -57,124 +87,178 @@ A MicroPython user C module is a directory with the following files:
 Basic example
 -------------
 
-This simple module named ``example`` provides a single function
-``example.add_ints(a, b)`` which adds the two integer args together and returns
-the result.
+This simple module named ``cexample`` provides a single function
+``cexample.add_ints(a, b)`` which adds the two integer args together and returns
+the result. It can be found in the MicroPython source tree
+`in the examples directory <https://github.com/micropython/micropython/tree/master/examples/usercmodule/cexample>`_
+and has a source file and a Makefile fragment with content as descibed above::
 
-Directory::
-
-    example/
-    ├── example.c
-    └── micropython.mk
+    micropython/
+    └──examples/
+       └──usercmodule/
+          └──cexample/
+             ├── examplemodule.c
+             ├── micropython.mk
+             └── micropython.cmake
 
 
-``example.c``
-
-.. code-block:: c
-
-    // Include required definitions first.
-    #include "py/obj.h"
-    #include "py/runtime.h"
-    #include "py/builtin.h"
-
-    // This is the function which will be called from Python as example.add_ints(a, b).
-    STATIC mp_obj_t example_add_ints(mp_obj_t a_obj, mp_obj_t b_obj) {
-        // Extract the ints from the micropython input objects
-        int a = mp_obj_get_int(a_obj);
-        int b = mp_obj_get_int(b_obj);
-
-        // Calculate the addition and convert to MicroPython object.
-        return mp_obj_new_int(a + b);
-    }
-    // Define a Python reference to the function above
-    STATIC MP_DEFINE_CONST_FUN_OBJ_2(example_add_ints_obj, example_add_ints);
-
-    // Define all properties of the example module.
-    // Table entries are key/value pairs of the attribute name (a string)
-    // and the MicroPython object reference.
-    // All identifiers and strings are written as MP_QSTR_xxx and will be
-    // optimized to word-sized integers by the build system (interned strings).
-    STATIC const mp_rom_map_elem_t example_module_globals_table[] = {
-        { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_example) },
-        { MP_ROM_QSTR(MP_QSTR_add_ints), MP_ROM_PTR(&example_add_ints_obj) },
-    };
-    STATIC MP_DEFINE_CONST_DICT(example_module_globals, example_module_globals_table);
-
-    // Define module object.
-    const mp_obj_module_t example_user_cmodule = {
-        .base = { &mp_type_module },
-        .globals = (mp_obj_dict_t*)&example_module_globals,
-    };
-
-    // Register the module to make it available in Python
-    MP_REGISTER_MODULE(MP_QSTR_example, example_user_cmodule, MODULE_EXAMPLE_ENABLED);
-
-
-``micropython.mk``
-
-.. code-block:: make
-
-    EXAMPLE_MOD_DIR := $(USERMOD_DIR)
-
-    # Add all C files to SRC_USERMOD.
-    SRC_USERMOD += $(EXAMPLE_MOD_DIR)/example.c
-
-    # We can add our module folder to include paths if needed
-    # This is not actually needed in this example.
-    CFLAGS_USERMOD += -I$(EXAMPLE_MOD_DIR)
-
-Finally you will need to define ``MODULE_EXAMPLE_ENABLED`` to 1. This
-can be done by adding ``CFLAGS_EXTRA=-DMODULE_EXAMPLE_ENABLED=1`` to
-the ``make`` command, or editing ``mpconfigport.h`` or
-``mpconfigboard.h`` to add
-
-.. code-block:: c
-
-    #define MODULE_EXAMPLE_ENABLED (1)
-
-Note that the exact method depends on the port as they have different
-structures. If not done correctly it will compile but importing will
-fail to find the module.
+Refer to the comments in these files for additional explanation.
+Next to the ``cexample`` module there's also ``cppexample`` which
+works in the same way but shows one way of mixing C and C++ code
+in MicroPython.
 
 
 Compiling the cmodule into MicroPython
 --------------------------------------
 
 To build such a module, compile MicroPython (see `getting started
-<https://github.com/micropython/micropython/wiki/Getting-Started>`_) with an
-extra ``make`` flag named ``USER_C_MODULES`` set to the directory containing
-all modules you want included (not to the module itself). For example:
+<https://github.com/micropython/micropython/wiki/Getting-Started>`_),
+applying 2 modifications:
 
+1. Set the build-time flag ``USER_C_MODULES`` to point to the modules
+   you want to include.  For ports that use Make this variable should be a
+   directory which is searched automatically for modules.  For ports that
+   use CMake this variable should be a file which includes the modules to
+   build.  See below for details.
 
-Directory::
+2. Enable the modules by setting the corresponding C preprocessor macro to
+   1.  This is only needed if the modules you are building are not
+   automatically enabled.
 
-    my_project/
-    ├── modules/
-    │   └──example/
-    │       ├──example.c
-    │       └──micropython.mk
-    └── micropython/
-        ├──ports/
-       ... ├──stm32/
-          ...
+For building the example modules which come with MicroPython,
+set ``USER_C_MODULES`` to the ``examples/usercmodule`` directory for Make,
+or to ``examples/usercmodule/micropython.cmake`` for CMake.
 
-Building for stm32 port:
+For example, here's how the to build the unix port with the example modules:
+
+.. code-block:: bash
+
+    cd micropython/ports/unix
+    make USER_C_MODULES=../../examples/usercmodule
+
+You may need to run ``make clean`` once at the start when including new
+user modules in the build.  The build output will show the modules found::
+
+    ...
+    Including User C Module from ../../examples/usercmodule/cexample
+    Including User C Module from ../../examples/usercmodule/cppexample
+    ...
+
+For a CMake-based port such as rp2, this will look a little different (note
+that CMake is actually invoked by ``make``):
+
+.. code-block:: bash
+
+    cd micropython/ports/rp2
+    make USER_C_MODULES=../../examples/usercmodule/micropython.cmake
+
+Again, you may need to run ``make clean`` first for CMake to pick up the
+user modules.  The CMake build output lists the modules by name::
+
+    ...
+    Including User C Module(s) from ../../examples/usercmodule/micropython.cmake
+    Found User C Module(s): usermod_cexample, usermod_cppexample
+    ...
+
+The contents of the top-level ``micropython.cmake`` can be used to control which
+modules are enabled.
+
+For your own projects it's more convenient to keep custom code out of the main
+MicroPython source tree, so a typical project directory structure will look
+like this::
+
+      my_project/
+      ├── modules/
+      │   ├── example1/
+      │   │   ├── example1.c
+      │   │   ├── micropython.mk
+      │   │   └── micropython.cmake
+      │   ├── example2/
+      │   │   ├── example2.c
+      │   │   ├── micropython.mk
+      │   │   └── micropython.cmake
+      │   └── micropython.cmake
+      └── micropython/
+          ├──ports/
+         ... ├──stm32/
+            ...
+
+When building with Make set ``USER_C_MODULES`` to the ``my_project/modules``
+directory.  For example, building the stm32 port:
 
 .. code-block:: bash
 
     cd my_project/micropython/ports/stm32
-    make USER_C_MODULES=../../../modules CFLAGS_EXTRA=-DMODULE_EXAMPLE_ENABLED=1 all
+    make USER_C_MODULES=../../../modules
+
+When building with CMake the top level ``micropython.cmake`` -- found directly
+in the ``my_project/modules`` directory -- should ``include`` all of the modules
+you want to have available:
+
+  .. code-block:: cmake
+
+      include(${CMAKE_CURRENT_LIST_DIR}/example1/micropython.cmake)
+      include(${CMAKE_CURRENT_LIST_DIR}/example2/micropython.cmake)
+
+Then build with:
+
+.. code-block:: bash
+
+    cd my_project/micropython/ports/esp32
+    make USER_C_MODULES=../../../../modules/micropython.cmake
+
+Note that the esp32 port needs the extra ``..`` for relative paths due to the
+location of its main ``CMakeLists.txt`` file.   You can also specify absolute
+paths to ``USER_C_MODULES``.
+
+All modules specified by the ``USER_C_MODULES`` variable (either found in this
+directory when using Make, or added via ``include`` when using CMake) will be
+compiled, but only those which are enabled will be available for importing.
+User modules are usually enabled by default (this is decided by the developer
+of the module), in which case there is nothing more to do than set ``USER_C_MODULES``
+as described above.
+
+If a module is not enabled by default then the corresponding C preprocessor macro
+must be enabled.  This macro name can be found by searching for the ``MP_REGISTER_MODULE``
+line in the module's source code (it usually appears at the end of the main source file).
+The third argument to ``MP_REGISTER_MODULE`` is the macro name, and this must be set
+to 1 using ``CFLAGS_EXTRA`` to make the module available.  If the third argument is just
+the number 1 then the module is enabled by default.
+
+For example, the ``examples/usercmodule/cexample`` module is enabled by default so
+has the following line in its source code:
+
+  .. code-block:: c
+
+      MP_REGISTER_MODULE(MP_QSTR_cexample, example_user_cmodule, 1);
+
+Alternatively, to make this module disabled by default but selectable through
+a preprocessor configuration option, it would be:
+
+  .. code-block:: c
+
+      MP_REGISTER_MODULE(MP_QSTR_cexample, example_user_cmodule, MODULE_CEXAMPLE_ENABLED);
+
+In this case the module is enabled by adding ``CFLAGS_EXTRA=-DMODULE_CEXAMPLE_ENABLED=1``
+to the ``make`` command, or editing ``mpconfigport.h`` or ``mpconfigboard.h`` to add
+
+  .. code-block:: c
+
+      #define MODULE_CEXAMPLE_ENABLED (1)
+
+Note that the exact method depends on the port as they have different
+structures.  If not done correctly it will compile but importing will
+fail to find the module.
 
 
 Module usage in MicroPython
 ---------------------------
 
-Once built into your copy of MicroPython, the module implemented
-in ``example.c`` above can now be accessed in Python just
-like any other builtin module, eg
+Once built into your copy of MicroPython, the module
+can now be accessed in Python just like any other builtin module, e.g.
 
 .. code-block:: python
 
-    import example
-    print(example.add_ints(1, 3))
+    import cexample
+    print(cexample.add_ints(1, 3))
     # should display 4

docs/develop/compiler.rst

@@ -0,0 +1,317 @@
+.. _compiler:
+
+The Compiler
+============
+
+The compilation process in MicroPython involves the following steps:
+
+* The lexer converts the stream of text that makes up a MicroPython program into tokens.
+* The parser then converts the tokens into an abstract syntax (parse tree).
+* Then bytecode or native code is emitted based on the parse tree.
+
+For purposes of this discussion we are going to add a simple language feature ``add1``
+that can be use in Python as:
+
+.. code-block:: bash
+
+    >>> add1 3
+    4
+    >>>
+
+The ``add1`` statement takes an integer as argument and adds ``1`` to it.
+
+Adding a grammar rule
+----------------------
+
+MicroPython's grammar is based on the `CPython grammar <https://docs.python.org/3.5/reference/grammar.html>`_
+and is defined in `py/grammar.h <https://github.com/micropython/micropython/blob/master/py/grammar.h>`_.
+This grammar is what is used to parse MicroPython source files.
+
+There are two macros you need to know to define a grammar rule: ``DEF_RULE`` and ``DEF_RULE_NC``.
+``DEF_RULE`` allows you to define a rule with an associated compile function,
+while ``DEF_RULE_NC`` has no compile (NC) function for it.
+
+A simple grammar definition with a compile function for our new ``add1`` statement
+looks like the following:
+
+.. code-block:: c
+
+   DEF_RULE(add1_stmt, c(add1_stmt), and(2), tok(KW_ADD1), rule(testlist))
+
+The second argument ``c(add1_stmt)`` is the corresponding compile function that should be implemented
+in ``py/compile.c`` to turn this rule into executable code.
+
+The third required argument can be ``or`` or ``and``. This specifies the number of nodes associated
+with a statement. For example, in this case, our ``add1`` statement is similar to ADD1 in assembly
+language. It takes one numeric argument. Therefore, the ``add1_stmt`` has two nodes associated with it.
+One node is for the statement itself, i.e the literal ``add1`` corresponding to ``KW_ADD1``,
+and the other for its argument, a ``testlist`` rule which is the top-level expression rule.
+
+.. note::
+   The ``add1`` rule here is just an example and not part of the standard
+   MicroPython grammar.
+
+The fourth argument in this example is the token associated with the rule, ``KW_ADD1``. This token should be
+defined in the lexer by editing ``py/lexer.h``.
+
+Defining the same rule without a compile function is achieved by using the ``DEF_RULE_NC`` macro
+and omitting the compile function argument:
+
+.. code-block:: c
+
+   DEF_RULE_NC(add1_stmt, and(2), tok(KW_ADD1), rule(testlist))
+
+The remaining arguments take on the same meaning. A rule without a compile function must
+be handled explicitly by all rules that may have this rule as a node. Such NC-rules are usually
+used to express sub-parts of a complicated grammar structure that cannot be expressed in a
+single rule.
+
+.. note::
+   The macros ``DEF_RULE`` and ``DEF_RULE_NC`` take other arguments. For an in-depth understanding of
+   supported parameters, see `py/grammar.h <https://github.com/micropython/micropython/blob/master/py/grammar.h>`_.
+
+Adding a lexical token
+----------------------
+
+Every rule defined in the grammar should have a token associated with it that is defined in ``py/lexer.h``.
+Add this token by editing the ``_mp_token_kind_t`` enum:
+
+.. code-block:: c
+   :emphasize-lines: 12
+
+   typedef enum _mp_token_kind_t {
+       ...
+       MP_TOKEN_KW_OR,
+       MP_TOKEN_KW_PASS,
+       MP_TOKEN_KW_RAISE,
+       MP_TOKEN_KW_RETURN,
+       MP_TOKEN_KW_TRY,
+       MP_TOKEN_KW_WHILE,
+       MP_TOKEN_KW_WITH,
+       MP_TOKEN_KW_YIELD,
+       MP_TOKEN_KW_ADD1,
+       ...
+   } mp_token_kind_t;
+
+Then also edit ``py/lexer.c`` to add the new keyword literal text:
+
+.. code-block:: c
+   :emphasize-lines: 12
+
+   STATIC const char *const tok_kw[] = {
+       ...
+       "or",
+       "pass",
+       "raise",
+       "return",
+       "try",
+       "while",
+       "with",
+       "yield",
+       "add1",
+       ...
+   };
+
+Notice the keyword is named depending on what you want it to be. For consistency, maintain the
+naming standard accordingly.
+
+.. note::
+   The order of these keywords in ``py/lexer.c`` must match the order of tokens in the enum
+   defined in ``py/lexer.h``.
+
+Parsing
+-------
+
+In the parsing stage the parser takes the tokens produced by the lexer and converts them to an abstract syntax tree (AST) or
+*parse tree*. The implementation for the parser is defined in `py/parse.c <https://github.com/micropython/micropython/blob/master/py/parse.c>`_.
+
+The parser also maintains a table of constants for use in different aspects of parsing, similar to what a
+`symbol table <https://steemit.com/programming/@drifter1/writing-a-simple-compiler-on-my-own-symbol-table-basic-structure>`_
+does.
+
+Several optimizations like `constant folding <http://compileroptimizations.com/category/constant_folding.htm>`_
+on integers for most operations e.g. logical, binary, unary, etc, and optimizing enhancements on parenthesis
+around expressions are performed during this phase, along with some optimizations on strings.
+
+It's worth noting that *docstrings* are discarded and not accessible to the compiler.
+Even optimizations like `string interning <https://en.wikipedia.org/wiki/String_interning>`_ are
+not applied to *docstrings*.
+
+Compiler passes
+---------------
+
+Like many compilers, MicroPython compiles all code to MicroPython bytecode or native code. The functionality
+that achieves this is implemented in `py/compile.c <https://github.com/micropython/micropython/blob/master/py/compile.c>`_.
+The most relevant method you should know about is this:
+
+.. code-block:: c
+
+   mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl) {
+       // Compile the input parse_tree to a raw-code structure.
+       mp_raw_code_t *rc = mp_compile_to_raw_code(parse_tree, source_file, is_repl);
+       // Create and return a function object that executes the outer module.
+       return mp_make_function_from_raw_code(rc, MP_OBJ_NULL, MP_OBJ_NULL);
+   }
+
+The compiler compiles the code in four passes: scope, stack size, code size and emit.
+Each pass runs the same C code over the same AST data structure, with different things
+being computed each time based on the results of the previous pass.
+
+First pass
+~~~~~~~~~~
+
+In the first pass, the compiler learns about the known identifiers (variables) and
+their scope, being global, local, closed over, etc. In the same pass the emitter
+(bytecode or native code) also computes the number of labels needed for the emitted
+code.
+
+.. code-block:: c
+
+   // Compile pass 1.
+   comp->emit = emit_bc;
+   comp->emit_method_table = &emit_bc_method_table;
+
+   uint max_num_labels = 0;
+   for (scope_t *s = comp->scope_head; s != NULL && comp->compile_error == MP_OBJ_NULL; s = s->next) {
+       if (s->emit_options == MP_EMIT_OPT_ASM) {
+           compile_scope_inline_asm(comp, s, MP_PASS_SCOPE);
+       } else {
+           compile_scope(comp, s, MP_PASS_SCOPE);
+
+           // Check if any implicitly declared variables should be closed over.
+           for (size_t i = 0; i < s->id_info_len; ++i) {
+               id_info_t *id = &s->id_info[i];
+               if (id->kind == ID_INFO_KIND_GLOBAL_IMPLICIT) {
+                   scope_check_to_close_over(s, id);
+               }
+           }
+       }
+       ...
+   }
+
+Second and third passes
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The second and third passes involve computing the Python stack size and code size
+for the bytecode or native code. After the third pass the code size cannot change,
+otherwise jump labels will be incorrect.
+
+.. code-block:: c
+
+   for (scope_t *s = comp->scope_head; s != NULL && comp->compile_error == MP_OBJ_NULL; s = s->next) {
+       ...
+
+       // Pass 2: Compute the Python stack size.
+       compile_scope(comp, s, MP_PASS_STACK_SIZE);
+
+       // Pass 3: Compute the code size.
+       if (comp->compile_error == MP_OBJ_NULL) {
+           compile_scope(comp, s, MP_PASS_CODE_SIZE);
+       }
+
+       ...
+   }
+
+Just before pass two there is a selection for the type of code to be emitted, which can
+either be native or bytecode.
+
+.. code-block:: c
+
+   // Choose the emitter type.
+   switch (s->emit_options) {
+       case MP_EMIT_OPT_NATIVE_PYTHON:
+       case MP_EMIT_OPT_VIPER:
+           if (emit_native == NULL) {
+               emit_native = NATIVE_EMITTER(new)(&comp->compile_error, &comp->next_label, max_num_labels);
+           }
+           comp->emit_method_table = NATIVE_EMITTER_TABLE;
+           comp->emit = emit_native;
+           break;
+
+       default:
+           comp->emit = emit_bc;
+           comp->emit_method_table = &emit_bc_method_table;
+           break;
+   }
+
+The bytecode option is the default but something unique to note for the native
+code option is that there is another option via ``VIPER``. See the
+:ref:`Emitting native code <emitting_native_code>` section for more details on
+viper annotations.
+
+There is also support for *inline assembly code*, where assembly instructions are
+written as Python function calls but are emitted directly as the corresponding
+machine code. This assembler has only three passes (scope, code size, emit)
+and uses a different implementation, not the ``compile_scope`` function.
+See the `inline assembler tutorial <https://docs.micropython.org/en/latest/pyboard/tutorial/assembler.html#pyboard-tutorial-assembler>`_
+for more details.
+
+Fourth pass
+~~~~~~~~~~~
+
+The fourth pass emits the final code that can be executed, either bytecode in
+the virtual machine, or native code directly by the CPU.
+
+.. code-block:: c
+
+   for (scope_t *s = comp->scope_head; s != NULL && comp->compile_error == MP_OBJ_NULL; s = s->next) {
+       ...
+
+       // Pass 4: Emit the compiled bytecode or native code.
+       if (comp->compile_error == MP_OBJ_NULL) {
+           compile_scope(comp, s, MP_PASS_EMIT);
+       }
+   }
+
+Emitting bytecode
+-----------------
+
+Statements in Python code usually correspond to emitted bytecode, for example ``a + b``
+generates "push a" then "push b" then "binary op add". Some statements do not emit
+anything but instead affect other things like the scope of variables, for example
+``global a``.
+
+The implementation of a function that emits bytecode looks similar to this:
+
+.. code-block:: c
+
+   void mp_emit_bc_unary_op(emit_t *emit, mp_unary_op_t op) {
+       emit_write_bytecode_byte(emit, 0, MP_BC_UNARY_OP_MULTI + op);
+   }
+
+We use the unary operator expressions for an example here but the implementation
+details are similar for other statements/expressions. The method ``emit_write_bytecode_byte()``
+is a wrapper around the main function ``emit_get_cur_to_write_bytecode()`` that all
+functions must call to emit bytecode.
+
+.. _emitting_native_code:
+
+Emitting native code
+---------------------
+
+Similar to how bytecode is generated, there should be a corresponding function in ``py/emitnative.c`` for each
+code statement:
+
+.. code-block:: c
+
+   STATIC void emit_native_unary_op(emit_t *emit, mp_unary_op_t op) {
+        vtype_kind_t vtype;
+        emit_pre_pop_reg(emit, &vtype, REG_ARG_2);
+        if (vtype == VTYPE_PYOBJ) {
+            emit_call_with_imm_arg(emit, MP_F_UNARY_OP, op, REG_ARG_1);
+            emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET);
+        } else {
+            adjust_stack(emit, 1);
+            EMIT_NATIVE_VIPER_TYPE_ERROR(emit,
+                MP_ERROR_TEXT("unary op %q not implemented"), mp_unary_op_method_name[op]);
+        }
+   }
+
+The difference here is that we have to handle *viper typing*. Viper annotations allow
+us to handle more than one type of variable. By default all variables are Python objects,
+but with viper a variable can also be declared as a machine-typed variable like a native
+integer or pointer. Viper can be thought of as a superset of Python, where normal Python
+objects are handled as usual, while native machine variables are handled in an optimised
+way by using direct machine instructions for the operations. Viper typing may break
+Python equivalence because, for example, integers become native integers and can overflow
+(unlike Python integers which extend automatically to arbitrary precision).

docs/develop/extendingmicropython.rst

@@ -0,0 +1,19 @@
+.. _extendingmicropython:
+
+Extending MicroPython in C
+==========================
+
+This chapter describes options for implementing additional functionality in C, but from code
+written outside of the main MicroPython repository. The first approach is useful for building
+your own custom firmware with some project-specific additional modules or functions that can
+be accessed from Python. The second approach is for building modules that can be loaded at runtime.
+
+Please see the :ref:`library section <internals_library>` for more information on building core modules that
+live in the main MicroPython repository.
+
+.. toctree::
+   :maxdepth: 3
+
+   cmodules.rst
+   natmod.rst
+   

docs/develop/gettingstarted.rst

@@ -0,0 +1,329 @@
+.. _gettingstarted:
+
+Getting Started
+===============
+
+This guide covers a step-by-step process on setting up version control, obtaining and building
+a copy of the source code for a port, building the documentation, running tests, and a description of the 
+directory structure of the MicroPython code base.
+
+Source control with git
+-----------------------
+
+MicroPython is hosted on `GitHub <https://github.com/micropython/micropython>`_ and uses
+`Git <https://git-scm.com>`_ for source control. The workflow is such that
+code is pulled and pushed to and from the main repository. Install the respective version
+of Git for your operating system to follow through the rest of the steps.
+
+.. note::
+   For a reference on the installation instructions, please refer to 
+   the `Git installation instructions <https://git-scm.com/book/en/v2/Getting-Started-Installing-Git>`_.
+   Learn about the basic git commands in this `Git Handbook <https://guides.github.com/introduction/git-handbook/>`_
+   or any other sources on the internet.
+
+.. note::
+   A .git-blame-ignore-revs file is included which avoids the output of git blame getting cluttered
+   by commits which are only for formatting code but have no functional changes. See `git blame documentation
+   <https://git-scm.com/docs/git-blame#Documentation/git-blame.txt---ignore-revltrevgt>`_ on how to use this.
+
+Get the code
+------------
+
+It is recommended that you maintain a fork of the MicroPython repository for your development purposes.
+The process of obtaining the source code includes the following:
+
+#. Fork the repository https://github.com/micropython/micropython
+#. You will now have a fork at <https://github.com/<your-user-name>/micropython>.
+#. Clone the forked repository using the following command:
+
+.. code-block:: bash
+
+   $ git clone https://github.com/<your-user-name>/micropython
+
+Then, `configure the remote repositories <https://git-scm.com/book/en/v2/Git-Basics-Working-with-Remotes>`_ to be able to
+collaborate on the MicroPython project.
+
+Configure remote upstream:
+
+.. code-block:: bash
+
+   $ cd micropython
+   $ git remote add upstream https://github.com/micropython/micropython
+
+It is common to configure ``upstream`` and ``origin`` on a forked repository
+to assist with sharing code changes. You can maintain your own mapping but
+it is recommended that ``origin`` maps to your fork and ``upstream`` to the main
+MicroPython repository.
+
+After the above configuration, your setup should be similar to this:
+
+.. code-block:: bash
+   
+   $ git remote -v
+   origin	https://github.com/<your-user-name>/micropython (fetch)
+   origin	https://github.com/<your-user-name>/micropython (push)
+   upstream	https://github.com/micropython/micropython (fetch)
+   upstream	https://github.com/micropython/micropython (push)
+
+You should now have a copy of the source code. By default, you are pointing
+to the master branch. To prepare for further development, it is recommended
+to work on a development branch.
+
+.. code-block:: bash
+
+    $ git checkout -b dev-branch
+
+You can give it any name. You will have to compile MicroPython whenever you change 
+to a different branch.
+
+Compile and build the code
+--------------------------
+
+When compiling MicroPython, you compile a specific :term:`port`, usually
+targeting a specific :ref:`board <glossary>`. Start by installing the required dependencies.
+Then build the MicroPython cross-compiler before you can successfully compile and build.
+This applies specifically when using Linux to compile.
+The Windows instructions are provided in a later section.
+
+.. _required_dependencies:
+
+Required dependencies
+~~~~~~~~~~~~~~~~~~~~~
+
+Install the required dependencies for Linux:
+
+.. code-block:: bash
+
+   $ sudo apt-get install build-essential libffi-dev git pkg-config
+
+For the stm32 port, the ARM cross-compiler is required:
+
+.. code-block:: bash
+
+   $ sudo apt-get install arm-none-eabi-gcc arm-none-eabi-binutils arm-none-eabi-newlib
+
+See the `ARM GCC
+toolchain <https://developer.arm.com/tools-and-software/open-source-software/developer-tools/gnu-toolchain/gnu-rm>`_
+for the latest details.
+
+Python is also required. Python 2 is supported for now, but we recommend using Python 3.
+Check that you have Python available on your system:
+
+.. code-block:: bash
+
+   $ python3
+   Python 3.5.0 (default, Jul 17 2020, 14:04:10) 
+   [GCC 5.4.0 20160609] on linux
+   Type "help", "copyright", "credits" or "license" for more information.
+   >>> 
+
+All supported ports have different dependency requirements, see their respective
+`readme files <https://github.com/micropython/micropython/tree/master/ports>`_.
+
+Building the MicroPython cross-compiler
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Almost all ports require building ``mpy-cross`` first to perform pre-compilation
+of Python code that will be included in the port firmware:
+
+.. code-block:: bash
+
+   $ cd mpy-cross
+   $ make
+
+.. note::
+   Note that, ``mpy-cross`` must be built for the host architecture
+   and not the target architecture.
+
+If it built successfully, you should see a message similar to this:
+
+.. code-block:: bash
+
+   LINK mpy-cross
+      text	   data	    bss	    dec	    hex	filename
+    279328	    776	    880	 280984	  44998	mpy-cross
+
+.. note::
+
+   Use ``make -C mpy-cross`` to build the cross-compiler in one statement
+   without moving to the ``mpy-cross`` directory otherwise, you will need
+   to do ``cd ..`` for the next steps.
+
+Building the Unix port of MicroPython
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The Unix port is a version of MicroPython that runs on Linux, macOS, and other Unix-like operating systems.
+It's extremely useful for developing MicroPython as it avoids having to deploy your code to a device to test it.
+In many ways, it works a lot like CPython's python binary.
+
+To build for the Unix port, make sure all Linux related dependencies are installed as detailed in the
+required dependencies section. See the :ref:`required_dependencies`
+to make sure that all dependencies are installed for this port. Also, make sure you have a working
+environment for ``gcc`` and ``GNU make``. Ubuntu 20.04 has been used for the example
+below but other unixes ought to work with little modification:
+
+.. code-block:: bash
+
+   $ gcc --version
+   gcc (Ubuntu 9.3.0-10ubuntu2) 9.3.0
+   Copyright (C) 2019 Free Software Foundation, Inc.
+   This is free software; see the source for copying conditions.  There is NO
+   warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.then build:
+
+.. code-block:: bash
+
+   $ cd ports/unix
+   $ make submodules
+   $ make
+
+If MicroPython built correctly, you should see the following:
+
+.. code-block:: bash
+
+   LINK micropython
+      text	   data	    bss	    dec	    hex	filename
+    412033	   5680	   2496	 420209	  66971	micropython
+
+Now run it:
+
+.. code-block:: bash
+
+   $ ./micropython
+   MicroPython v1.13-38-gc67012d-dirty on 2020-09-13; linux version
+   Use Ctrl-D to exit, Ctrl-E for paste mode
+   >>> print("hello world")
+   hello world
+   >>>
+
+Building the Windows port
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The Windows port includes a Visual Studio project file micropython.vcxproj that you can use to build micropython.exe.
+It can be opened in Visual Studio or built from the command line using msbuild. Alternatively, it can be built using mingw,
+either in Windows with Cygwin, or on Linux.
+See `windows port documentation <https://github.com/micropython/micropython/tree/master/ports/windows>`_ for more information.
+
+Building the STM32 port
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Like the Unix port, you need to install some required dependencies
+as detailed in the :ref:`required_dependencies` section, then build:
+
+.. code-block:: bash
+
+   $ cd ports/stm32
+   $ make submodules
+   $ make
+
+Please refer to the `stm32 documentation <https://github.com/micropython/micropython/tree/master/ports/stm32>`_ 
+for more details on flashing the firmware.
+
+.. note::
+   See the :ref:`required_dependencies` to make sure that all dependencies are installed for this port.
+   The cross-compiler is needed. ``arm-none-eabi-gcc`` should also be in the $PATH or specified manually
+   via CROSS_COMPILE, either by setting the environment variable or in the ``make`` command line arguments.
+
+You can also specify which board to use:
+
+.. code-block:: bash
+
+   $ cd ports/stm32
+   $ make submodules
+   $ make BOARD=<board>
+
+See `ports/stm32/boards <https://github.com/micropython/micropython/tree/master/ports/stm32/boards>`_
+for the available boards. e.g. "PYBV11" or "NUCLEO_WB55".
+
+Building the documentation
+--------------------------
+
+MicroPython documentation is created using ``Sphinx``. If you have already
+installed Python, then install ``Sphinx`` using ``pip``. It is recommended
+that you use a virtual environment:
+
+.. code-block:: bash
+
+   $ python3 -m venv env
+   $ source env/bin/activate
+   $ pip install sphinx
+
+Navigate to the ``docs`` directory:
+
+.. code-block:: bash
+
+   $ cd docs
+
+Build the docs:
+
+.. code-block:: bash
+
+   $ make html
+
+Open ``docs/build/html/index.html`` in your browser to view the docs locally. Refer to the 
+documentation on `importing your documentation
+<https://docs.readthedocs.io/en/stable/intro/import-guide.html>`_ to use Read the Docs.
+
+Running the tests
+-----------------
+
+To run all tests in the test suite on the Unix port use:
+
+.. code-block:: bash
+
+   $ cd ports/unix
+   $ make test
+
+To run a selection of tests on a board/device connected over USB use:
+
+.. code-block:: bash
+
+   $ cd tests
+   $ ./run-tests.py --target minimal --device /dev/ttyACM0
+
+See also :ref:`writingtests`.
+
+Folder structure
+----------------
+
+There are a couple of directories to take note of in terms of where certain implementation details
+are. The following is a break down of the top-level folders in the source code.
+
+py
+
+  Contains the compiler, runtime, and core library implementation.
+
+mpy-cross
+
+  Has the MicroPython cross-compiler which pre-compiles the Python scripts to bytecode.
+
+ports
+
+  Code for all the versions of MicroPython for the supported ports.
+
+lib
+
+  Low-level C libraries used by any port which are mostly 3rd-party libraries.
+
+drivers
+
+  Has drivers for specific hardware and intended to work across multiple ports.
+
+extmod
+
+  Contains a C implementation of more non-core modules.
+
+docs
+
+  Has the standard documentation found at https://docs.micropython.org/.
+
+tests
+
+  An implementation of the test suite.
+
+tools
+
+  Contains helper tools including the ``upip`` and the ``pyboard.py`` module.
+
+examples
+
+  Example code for building MicroPython as a library as well as native modules.

docs/develop/index.rst

@@ -1,14 +1,27 @@
-Developing and building MicroPython
-===================================
+MicroPython Internals
+=====================
 
-This chapter describes some options for extending MicroPython in C. Note
-that it doesn't aim to be a complete guide for developing with MicroPython.
-See the `getting started guide
-<https://github.com/micropython/micropython/wiki/Getting-Started>`_ for further information.
+This chapter covers a tour of MicroPython from the perspective of a developer, contributing
+to MicroPython. It acts as a comprehensive resource on the implementation details of MicroPython
+for both novice and expert contributors.
+
+Development around MicroPython usually involves modifying the core runtime, porting or 
+maintaining a new library. This guide describes at great depth, the implementation
+details of MicroPython including a getting started guide, compiler internals, porting
+MicroPython to a new platform and implementing a core MicroPython library.
 
 .. toctree::
-   :maxdepth: 1
+   :maxdepth: 3
 
-   cmodules.rst
+   gettingstarted.rst
+   writingtests.rst
+   compiler.rst
+   memorymgt.rst
+   library.rst
+   optimizations.rst
    qstr.rst
-   natmod.rst
+   maps.rst
+   publiccapi.rst
+   extendingmicropython.rst
+   porting.rst
+   

docs/develop/library.rst

@@ -0,0 +1,86 @@
+.. _internals_library:
+
+Implementing a Module
+=====================
+
+This chapter details how to implement a core module in MicroPython.
+MicroPython modules can be one of the following:
+
+- Built-in module: A general module that is be part of the MicroPython repository.
+- User module: A module that is useful for your specific project that you maintain
+  in your own repository or private codebase.
+- Dynamic module: A module that can be deployed and imported at runtime to your device.
+
+A module in MicroPython can be implemented in one of the following locations:
+
+- py/: A core library that mirrors core CPython functionality.
+- extmod/: A CPython or MicroPython-specific module that is shared across multiple ports.
+- ports/<port>/: A port-specific module.
+
+.. note::
+   This chapter describes modules implemented in ``py/`` or core modules.
+   See :ref:`extendingmicropython` for details on implementing an external module.
+   For details on port-specific modules, see :ref:`porting_to_a_board`.
+
+Implementing a core module
+--------------------------
+
+Like CPython, MicroPython has core builtin modules that can be accessed through import statements.
+An example is the ``gc`` module discussed in :ref:`memorymanagement`.
+
+.. code-block:: bash
+
+   >>> import gc
+   >>> gc.enable()
+   >>>
+
+MicroPython has several other builtin standard/core modules like ``io``, ``uarray`` etc.
+Adding a new core module involves several modifications.
+
+First, create the ``C`` file in the ``py/`` directory. In this example we are adding a
+hypothetical new module ``subsystem`` in the file ``modsubsystem.c``:
+
+.. code-block:: c
+
+   #include "py/builtin.h"
+   #include "py/runtime.h"
+
+   #if MICROPY_PY_SUBSYSTEM
+
+   // info()
+   STATIC mp_obj_t py_subsystem_info(void) {
+       return MP_OBJ_NEW_SMALL_INT(42);
+   }
+   MP_DEFINE_CONST_FUN_OBJ_0(subsystem_info_obj, py_subsystem_info);
+
+   STATIC const mp_rom_map_elem_t mp_module_subsystem_globals_table[] = {
+       { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_subsystem) },
+       { MP_ROM_QSTR(MP_QSTR_info), MP_ROM_PTR(&subsystem_info_obj) },
+   };
+   STATIC MP_DEFINE_CONST_DICT(mp_module_subsystem_globals, mp_module_subsystem_globals_table);
+
+   const mp_obj_module_t mp_module_subsystem = {
+       .base = { &mp_type_module },
+       .globals = (mp_obj_dict_t *)&mp_module_subsystem_globals,
+   };
+
+   MP_REGISTER_MODULE(MP_QSTR_subsystem, mp_module_subsystem, MICROPY_PY_SUBSYSTEM);
+
+   #endif
+
+The implementation includes a definition of all functions related to the module and adds the
+functions to the module's global table in ``mp_module_subsystem_globals_table``. It also
+creates the module object with ``mp_module_subsystem``.  The module is then registered with
+the wider system via the ``MP_REGISTER_MODULE`` macro.
+
+After building and running the modified MicroPython, the module should now be importable:
+
+.. code-block:: bash
+
+   >>> import subsystem
+   >>> subsystem.info()
+   42
+   >>>
+
+Our ``info()`` function currently returns just a single number but can be extended
+to do anything.  Similarly, more functions can be added to this new module.

docs/develop/maps.rst

@@ -0,0 +1,63 @@
+.. _maps:
+
+Maps and Dictionaries
+=====================
+
+MicroPython dictionaries and maps use techniques called open addressing and linear probing.
+This chapter details both of these methods.
+
+Open addressing
+---------------
+
+`Open addressing <https://en.wikipedia.org/wiki/Open_addressing>`_ is used to resolve collisions.
+Collisions are very common occurrences and happen when two items happen to hash to the same
+slot or location. For example, given a hash setup as this:
+
+.. image:: img/collision.png
+
+If there is a request to fill slot ``0`` with ``70``, since the slot ``0`` is not empty, open addressing
+finds the next available slot in the dictionary to service this request. This sequential search for an alternate
+location is called *probing*. There are several sequence probing algorithms but MicroPython uses
+linear probing that is described in the next section.
+
+Linear probing
+--------------
+
+Linear probing is one of the methods for finding an available address or slot in a dictionary. In MicroPython,
+it is used with open addressing. To service the request described above, unlike other probing algorithms,
+linear probing assumes a fixed interval of ``1`` between probes. The request will therefore be serviced by
+placing the item in the next free slot which is slot ``4`` in our example:
+
+.. image:: img/linprob.png
+
+The same methods i.e open addressing and linear probing are used to search for an item in a dictionary.
+Assume we want to search for the data item ``33``. The computed hash value will be 2. Looking at slot 2
+reveals ``33``, at this point, we return ``True``. Searching for ``70`` is quite different as there was a
+collision at the time of insertion. Therefore computing the hash value is ``0`` which is currently
+holding ``44``. Instead of simply returning ``False``, we perform a sequential search starting at point
+``1`` until the item ``70`` is found or we encounter a free slot. This is the general way of performing
+look-ups in hashes:
+
+.. code-block:: c
+
+   // not yet found, keep searching in this table
+   pos = (pos + 1) % set->alloc;
+
+   if (pos == start_pos) {
+       // search got back to starting position, so index is not in table
+       if (lookup_kind & MP_MAP_LOOKUP_ADD_IF_NOT_FOUND) {
+           if (avail_slot != NULL) {
+               // there was an available slot, so use that
+               set->used++;
+               *avail_slot = index;
+               return index;
+           } else {
+               // not enough room in table, rehash it
+               mp_set_rehash(set);
+               // restart the search for the new element
+               start_pos = pos = hash % set->alloc;
+           }
+       }
+   } else {
+        return MP_OBJ_NULL;
+   }

docs/develop/memorymgt.rst

@@ -0,0 +1,141 @@
+.. _memorymanagement:
+
+Memory Management
+=================
+
+Unlike programming languages such as C/C++, MicroPython hides memory management
+details from the developer by supporting automatic memory management.
+Automatic memory management is a technique used by operating systems or applications to automatically manage
+the allocation and deallocation of memory. This eliminates challenges such as forgetting to
+free the memory allocated to an object. Automatic memory management also avoids the critical issue of using memory
+that is already released. Automatic memory management takes many forms, one of them being
+garbage collection (GC).
+
+The garbage collector usually has two responsibilities;
+
+#. Allocate new objects in available memory.
+#. Free unused memory.
+
+There are many GC algorithms but MicroPython uses the
+`Mark and Sweep <https://en.wikipedia.org/wiki/Tracing_garbage_collection#Basic_algorithm>`_
+policy for managing memory. This algorithm has a mark phase that traverses the heap marking all
+live objects while the sweep phase goes through the heap reclaiming all unmarked objects.
+
+Garbage collection functionality in MicroPython is available through the ``gc`` built-in
+module:
+
+.. code-block:: bash
+
+   >>> x = 5
+   >>> x
+   5
+   >>> import gc
+   >>> gc.enable()
+   >>> gc.mem_alloc()
+   1312
+   >>> gc.mem_free()
+   2071392
+   >>> gc.collect()
+   19
+   >>> gc.disable()
+   >>>
+
+Even when ``gc.disable()`` is invoked, collection can be triggered with ``gc.collect()``.
+
+The object model
+----------------
+
+All MicroPython objects are referred to by the ``mp_obj_t`` data type.
+This is usually word-sized (i.e. the same size as a pointer on the target architecture),
+and can be typically 32-bit (STM32, nRF, ESP32, Unix x86) or 64-bit (Unix x64).
+It can also be greater than a word-size for certain object representations, for
+example ``OBJ_REPR_D`` has a 64-bit sized ``mp_obj_t`` on a 32-bit architecture.
+
+An ``mp_obj_t`` represents a MicroPython object, for example an integer, float, type, dict or
+class instance. Some objects, like booleans and small integers, have their value stored directly
+in the ``mp_obj_t`` value and do not require additional memory. Other objects have their value
+store elsewhere in memory (for example on the garbage-collected heap) and their ``mp_obj_t`` contains
+a pointer to that memory. A portion of ``mp_obj_t`` is the tag which tells what type of object it is.
+
+See ``py/mpconfig.h`` for the specific details of the available representations.
+
+**Pointer tagging**
+
+Because pointers are word-aligned, when they are stored in an ``mp_obj_t`` the
+lower bits of this object handle will be zero.  For example on a 32-bit architecture
+the lower 2 bits will be zero:
+
+``********|********|********|******00``
+
+These bits are reserved for purposes of storing a tag. The tag stores extra information as
+opposed to introducing a new field to store that information in the object, which may be
+inefficient.  In MicroPython the tag tells if we are dealing with a small integer, interned
+(small) string or a concrete object, and different semantics apply to each of these.
+
+For small integers the mapping is this:
+
+``********|********|********|*******1``
+
+Where the asterisks hold the actual integer value.  For an interned string or an immediate
+object (e.g. ``True``) the layout of the ``mp_obj_t`` value is, respectively:
+
+``********|********|********|*****010``
+
+``********|********|********|*****110``
+
+While a concrete object that is none of the above takes the form:
+
+``********|********|********|******00``
+
+The stars here correspond to the address of the concrete object in memory.
+
+Allocation of objects
+----------------------
+
+The value of a small integer is stored directly in the ``mp_obj_t`` and will be
+allocated in-place, not on the heap or elsewhere.  As such, creation of small
+integers does not affect the heap.  Similarly for interned strings that already have
+their textual data stored elsewhere, and immediate values like ``None``, ``False``
+and ``True``.
+
+Everything else which is a concrete object is allocated on the heap and its object structure is such that
+a field is reserved in the object header to store the type of the object.
+
+.. code-block:: bash
+
+    +++++++++++
+    +         +
+    + type    + object header
+    +         +
+    +++++++++++
+    +         + object items
+    +         +
+    +         +
+    +++++++++++
+
+The heap's smallest unit of allocation is a block, which is four machine words in
+size (16 bytes on a 32-bit machine, 32 bytes on a 64-bit machine).
+Another structure also allocated on the heap tracks the allocation of
+objects in each block. This structure is called a *bitmap*.
+
+.. image:: img/bitmap.png
+
+The bitmap tracks whether a block is "free" or "in use" and use two bits to track this state
+for each block.
+
+The mark-sweep garbage collector manages the objects allocated on the heap, and also
+utilises the bitmap to mark objects that are still in use.
+See `py/gc.c <https://github.com/micropython/micropython/blob/master/py/gc.c>`_
+for the full implementation of these details.
+
+**Allocation: heap layout**
+
+The heap is arranged such that it consists of blocks in pools. A block
+can have different properties:
+
+- *ATB(allocation table byte):* If set, then the block is a normal block
+- *FREE:* Free block
+- *HEAD:* Head of a chain of blocks
+- *TAIL:* In the tail of a chain of blocks
+- *MARK :* Marked head block
+- *FTB(finaliser table byte):* If set, then the block has a finaliser

docs/develop/natmod.rst

@@ -21,7 +21,8 @@ language which can be compiled to stand-alone machine code can be put into a
 
 A native .mpy module is built using the ``mpy_ld.py`` tool, which is found in the
 ``tools/`` directory of the project.  This tool takes a set of object files
-(.o files) and links them together to create a native .mpy files.
+(.o files) and links them together to create a native .mpy files.  It requires
+CPython 3 and the library pyelftools v0.25 or greater.
 
 Supported features and limitations
 ----------------------------------
@@ -179,6 +180,14 @@ The file ``Makefile`` contains:
 Compiling the module
 --------------------
 
+The prerequisite tools needed to build a native .mpy file are:
+
+* The MicroPython repository (at least the ``py/`` and ``tools/`` directories).
+* CPython 3, and the library pyelftools (eg ``pip install 'pyelftools>=0.25'``).
+* GNU make.
+* A C compiler for the target architecture (if C source is used).
+* Optionally ``mpy-cross``, built from the MicroPython repository (if .py source is used).
+
 Be sure to select the correct ``ARCH`` for the target you are going to run on.
 Then build with::
 
@@ -193,7 +202,7 @@ Module usage in MicroPython
 
 Once the module is built there should be a file called ``factorial.mpy``.  Copy
 this so it is accessible on the filesystem of your MicroPython system and can be
-found in the import path.  The module con now be accessed in Python just like any
+found in the import path.  The module can now be accessed in Python just like any
 other module, for example::
 
     import factorial

docs/develop/optimizations.rst

@@ -0,0 +1,72 @@
+.. _optimizations:
+
+Optimizations
+=============
+
+MicroPython uses several optimizations to save RAM but also ensure the efficient
+execution of programs. This chapter discusses some of these optimizations.
+
+.. note::
+   :ref:`qstr` and :ref:`maps` details other optimizations on strings and
+   dictionaries.
+
+Frozen bytecode
+---------------
+
+When MicroPython loads Python code from the filesystem, it first has to parse the file into
+a temporary in-memory representation, and then generate bytecode for execution, both of which
+are stored in the heap (in RAM). This can lead to significant amounts of memory being used.
+The MicroPython cross compiler can be used to generate
+a ``.mpy`` file, containing the pre-compiled bytecode for a Python module. This will still
+be loaded into RAM, but it avoids the additional overhead of the parsing stage.
+
+As a further optimisation, the pre-compiled bytecode from a ``.mpy`` file can be "frozen"
+into the firmware image as part of the main firmware compilation process, which means that
+the bytecode will be executed from ROM. This can lead to a significant memory saving, and
+reduce heap fragmentation.
+
+Variables
+---------
+
+MicroPython processes local and global variables differently. Global variables
+are stored and looked up from a global dictionary that is allocated on the heap
+(note that each module has its own separate dict, so separate namespace).
+Local variables on the other hand are are stored on the Python value stack, which may
+live on the C stack or on the heap.  They are accessed directly by their offset
+within the Python stack, which is more efficient than a global lookup in a dict.
+
+The length of global variable names also affects how much RAM is used as identifiers
+are stored in RAM. The shorter the identifier, the less memory is used.
+
+The other aspect is that ``const`` variables that start with an underscore are treated as
+proper constants and are not allocated or added in a dictionary, hence saving some memory.
+These variables use ``const()`` from the MicroPython library. Therefore:
+
+.. code-block:: python
+
+    from micropython import const
+
+    X = const(1)
+    _Y = const(2)
+    foo(X, _Y)
+
+Compiles to:
+
+.. code-block:: python
+
+    X = 1
+    foo(1, 2)
+
+Allocation of memory
+--------------------
+
+Most of the common MicroPython constructs are not allocated on the heap.
+However the following are:
+
+- Dynamic data structures like lists, mappings, etc;
+- Functions, classes and object instances;
+- imports; and
+- First-time assignment of global variables (to create the slot in the global dict).
+
+For a detailed discussion on a more user-centric perspective on optimization,
+see `Maximising MicroPython speed <https://docs.micropython.org/en/latest/reference/speed_python.html>`_

docs/develop/porting.rst

@@ -0,0 +1,310 @@
+.. _porting_to_a_board:
+
+Porting MicroPython
+===================
+
+The MicroPython project contains several ports to different microcontroller families and
+architectures. The project repository has a `ports <https://github.com/micropython/micropython/tree/master/ports>`_
+directory containing a subdirectory for each supported port.
+
+A port will typically contain definitions for multiple "boards", each of which is a specific piece of
+hardware that that port can run on, e.g. a development kit or device.
+
+The `minimal port <https://github.com/micropython/micropython/tree/master/ports/minimal>`_ is
+available as a simplified reference implementation of a MicroPython port.  It can run on both the
+host system and an STM32F4xx MCU.
+
+In general, starting a port requires:
+
+- Setting up the toolchain (configuring Makefiles, etc).
+- Implementing boot configuration and CPU initialization.
+- Initialising basic drivers required for development and debugging (e.g. GPIO, UART).
+- Performing the board-specific configurations.
+- Implementing the port-specific modules.
+
+Minimal MicroPython firmware
+----------------------------
+
+The best way to start porting MicroPython to a new board is by integrating a minimal
+MicroPython interpreter.  For this walkthrough, create a subdirectory for the new
+port in the ``ports`` directory:
+
+.. code-block:: bash
+
+   $ cd ports
+   $ mkdir example_port
+
+The basic MicroPython firmware is implemented in the main port file, e.g ``main.c``:
+
+.. code-block:: c
+
+   #include "py/compile.h"
+   #include "py/gc.h"
+   #include "py/mperrno.h"
+   #include "py/stackctrl.h"
+   #include "lib/utils/gchelper.h"
+   #include "lib/utils/pyexec.h"
+
+   // Allocate memory for the MicroPython GC heap.
+   static char heap[4096];
+
+   int main(int argc, char **argv) {
+       // Initialise the MicroPython runtime.
+       mp_stack_ctrl_init();
+       gc_init(heap, heap + sizeof(heap));
+       mp_init();
+       mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_path), 0);
+       mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_argv), 0);
+
+       // Start a normal REPL; will exit when ctrl-D is entered on a blank line.
+       pyexec_friendly_repl();
+
+       // Deinitialise the runtime.
+       gc_sweep_all();
+       mp_deinit();
+       return 0;
+   }
+
+   // Handle uncaught exceptions (should never be reached in a correct C implementation).
+   void nlr_jump_fail(void *val) {
+       for (;;) {
+       }
+   }
+
+   // Do a garbage collection cycle.
+   void gc_collect(void) {
+       gc_collect_start();
+       gc_helper_collect_regs_and_stack();
+       gc_collect_end();
+   }
+
+   // There is no filesystem so stat'ing returns nothing.
+   mp_import_stat_t mp_import_stat(const char *path) {
+       return MP_IMPORT_STAT_NO_EXIST;
+   }
+
+   // There is no filesystem so opening a file raises an exception.
+   mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
+       mp_raise_OSError(MP_ENOENT);
+   }
+
+We also need a Makefile at this point for the port:
+
+.. code-block:: Makefile
+
+   # Include the core environment definitions; this will set $(TOP).
+   include ../../py/mkenv.mk
+
+   # Include py core make definitions.
+   include $(TOP)/py/py.mk
+
+   # Set CFLAGS and libraries.
+   CFLAGS = -I. -I$(BUILD) -I$(TOP)
+   LIBS = -lm
+
+   # Define the required source files.
+   SRC_C = \
+       main.c \
+       mphalport.c \
+       lib/mp-readline/readline.c \
+       lib/utils/gchelper_generic.c \
+       lib/utils/pyexec.c \
+       lib/utils/stdout_helpers.c \
+
+   # Define the required object files.
+   OBJ = $(PY_CORE_O) $(addprefix $(BUILD)/, $(SRC_C:.c=.o))
+
+   # Define the top-level target, the main firmware.
+   all: $(BUILD)/firmware.elf
+
+   # Define how to build the firmware.
+   $(BUILD)/firmware.elf: $(OBJ)
+       $(ECHO) "LINK $@"
+       $(Q)$(CC) $(LDFLAGS) -o $@ $^ $(LIBS)
+       $(Q)$(SIZE) $@
+
+   # Include remaining core make rules.
+   include $(TOP)/py/mkrules.mk
+
+Remember to use proper tabs to indent the Makefile.
+
+MicroPython Configurations
+--------------------------
+
+After integrating the minimal code above, the next step is to create the MicroPython
+configuration files for the port. The compile-time configurations are specified in
+``mpconfigport.h`` and additional hardware-abstraction functions, such as time keeping,
+in ``mphalport.h``.
+
+The following is an example of an ``mpconfigport.h`` file:
+
+.. code-block:: c
+
+   #include <stdint.h>
+
+   // Python internal features.
+   #define MICROPY_ENABLE_GC                       (1)
+   #define MICROPY_HELPER_REPL                     (1)
+   #define MICROPY_ERROR_REPORTING                 (MICROPY_ERROR_REPORTING_TERSE)
+   #define MICROPY_FLOAT_IMPL                      (MICROPY_FLOAT_IMPL_FLOAT)
+
+   // Fine control over Python builtins, classes, modules, etc.
+   #define MICROPY_PY_ASYNC_AWAIT                  (0)
+   #define MICROPY_PY_BUILTINS_SET                 (0)
+   #define MICROPY_PY_ATTRTUPLE                    (0)
+   #define MICROPY_PY_COLLECTIONS                  (0)
+   #define MICROPY_PY_MATH                         (0)
+   #define MICROPY_PY_IO                           (0)
+   #define MICROPY_PY_STRUCT                       (0)
+
+   // Type definitions for the specific machine.
+
+   typedef intptr_t mp_int_t; // must be pointer size
+   typedef uintptr_t mp_uint_t; // must be pointer size
+   typedef long mp_off_t;
+
+   // We need to provide a declaration/definition of alloca().
+   #include <alloca.h>
+
+   // Define the port's name and hardware.
+   #define MICROPY_HW_BOARD_NAME "example-board"
+   #define MICROPY_HW_MCU_NAME   "unknown-cpu"
+
+   #define MP_STATE_PORT MP_STATE_VM
+
+   #define MICROPY_PORT_ROOT_POINTERS \
+       const char *readline_hist[8];
+
+This configuration file contains machine-specific configurations including aspects like if different
+MicroPython features should be enabled e.g. ``#define MICROPY_ENABLE_GC (1)``. Making this Setting
+``(0)`` disables the feature.
+
+Other configurations include type definitions, root pointers, board name, microcontroller name
+etc.
+
+Similarly, an minimal example ``mphalport.h`` file looks like this:
+
+.. code-block:: c
+
+   static inline void mp_hal_set_interrupt_char(char c) {}
+
+Support for standard input/output
+---------------------------------
+
+MicroPython requires at least a way to output characters, and to have a REPL it also
+requires a way to input characters. Functions for this can be implemented in the file
+``mphalport.c``, for example:
+
+.. code-block:: c
+
+   #include <unistd.h>
+   #include "py/mpconfig.h"
+
+   // Receive single character, blocking until one is available.
+   int mp_hal_stdin_rx_chr(void) {
+       unsigned char c = 0;
+       int r = read(STDIN_FILENO, &c, 1);
+       (void)r;
+       return c;
+   }
+
+   // Send the string of given length.
+   void mp_hal_stdout_tx_strn(const char *str, mp_uint_t len) {
+       int r = write(STDOUT_FILENO, str, len);
+       (void)r;
+   }
+
+These input and output functions have to be modified depending on the
+specific board API. This example uses the standard input/output stream.
+
+Building and running
+--------------------
+
+At this stage the directory of the new port should contain::
+
+    ports/example_port/
+    ├── main.c
+    ├── Makefile
+    ├── mpconfigport.h
+    ├── mphalport.c
+    └── mphalport.h
+
+The port can now be built by running ``make`` (or otherwise, depending on your system).
+
+If you are using the default compiler settings in the Makefile given above then this
+will create an executable called ``build/firmware.elf`` which can be executed directly.
+To get a functional REPL you may need to first configure the terminal to raw mode:
+
+.. code-block:: bash
+
+   $ stty raw opost -echo
+   $ ./build/firmware.elf
+
+That should give a MicroPython REPL.  You can then run commands like:
+
+.. code-block:: bash
+
+   MicroPython v1.13 on 2021-01-01; example-board with unknown-cpu
+   >>> import usys
+   >>> usys.implementation
+   ('micropython', (1, 13, 0))
+   >>>
+
+Use Ctrl-D to exit, and then run ``reset`` to reset the terminal.
+
+Adding a module to the port
+---------------------------
+
+To add a custom module like ``myport``, first add the module definition in a file
+``modmyport.c``:
+
+.. code-block:: c
+
+   #include "py/runtime.h"
+
+   STATIC mp_obj_t myport_info(void) {
+       mp_printf(&mp_plat_print, "info about my port\n");
+       return mp_const_none;
+   }
+   STATIC MP_DEFINE_CONST_FUN_OBJ_0(myport_info_obj, myport_info);
+
+   STATIC const mp_rom_map_elem_t myport_module_globals_table[] = {
+       { MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(MP_QSTR_myport) },
+       { MP_ROM_QSTR(MP_QSTR_info), MP_ROM_PTR(&myport_info_obj) },
+   };
+   STATIC MP_DEFINE_CONST_DICT(myport_module_globals, myport_module_globals_table);
+
+   const mp_obj_module_t myport_module = {
+       .base = { &mp_type_module },
+       .globals = (mp_obj_dict_t *)&myport_module_globals,
+   };
+
+   MP_REGISTER_MODULE(MP_QSTR_myport, myport_module, 1);
+
+Note: the "1" as the third argument in ``MP_REGISTER_MODULE`` enables this new module
+unconditionally. To allow it to be conditionally enabled, replace the "1" by
+``MICROPY_PY_MYPORT`` and then add ``#define MICROPY_PY_MYPORT (1)`` in ``mpconfigport.h``
+accordingly.
+
+You will also need to edit the Makefile to add ``modmyport.c`` to the ``SRC_C`` list, and
+a new line adding the same file to ``SRC_QSTR`` (so qstrs are searched for in this new file),
+like this:
+
+.. code-block:: Makefile
+
+   SRC_C = \
+       main.c \
+       modmyport.c \
+       mphalport.c \
+       ...
+
+   SRC_QSTR += modport.c
+
+If all went correctly then, after rebuilding, you should be able to import the new module:
+
+.. code-block:: bash
+
+    >>> import myport
+    >>> myport.info()
+    info about my port
+    >>>

docs/develop/publiccapi.rst

@@ -0,0 +1,25 @@
+.. _publiccapi:
+
+The public C API
+================
+
+The public C-API comprises functions defined in all C header files in the ``py/``
+directory. Most of the important core runtime C APIs are exposed in ``runtime.h`` and
+``obj.h``.
+
+The following is an example of public API functions from ``obj.h``:
+
+.. code-block:: c
+
+   mp_obj_t mp_obj_new_list(size_t n, mp_obj_t *items);
+   mp_obj_t mp_obj_list_append(mp_obj_t self_in, mp_obj_t arg);
+   mp_obj_t mp_obj_list_remove(mp_obj_t self_in, mp_obj_t value);
+   void mp_obj_list_get(mp_obj_t self_in, size_t *len, mp_obj_t **items);
+
+At its core, any functions and macros in header files make up the public
+API and can be used to access very low-level details of MicroPython. Static
+inline functions in header files are fine too, such functions will be
+inlined in the code when used.
+
+Header files in the ``ports`` directory are only exposed to the functionality
+specific to a given port.

docs/develop/qstr.rst

@@ -1,3 +1,5 @@
+.. _qstr:
+
 MicroPython string interning
 ============================
 
@@ -51,12 +53,13 @@ Processing happens in the following stages:
    through the C pre-processor.  This means that any conditionally disabled code
    will be removed, and macros expanded.  This means we don't add strings to the
    pool that won't be used in the final firmware.  Because at this stage (thanks
-   to the ``NO_QSTR`` macro added by ``QSTR_GEN_EXTRA_CFLAGS``) there is no
+   to the ``NO_QSTR`` macro added by ``QSTR_GEN_CFLAGS``) there is no
    definition for ``MP_QSTR_Foo`` it passes through this stage unaffected.  This
    file also includes comments from the preprocessor that include line number
    information.  Note that this step only uses files that have changed, which
    means that ``qstr.i.last`` will only contain data from files that have
    changed since the last compile.
+   
 2. ``qstr.split`` is an empty file created after running ``makeqstrdefs.py split``
    on qstr.i.last. It's just used as a dependency to indicate that the step ran.
    This script outputs one file per input C file,  ``genhdr/qstr/...file.c.qstr``,
@@ -71,8 +74,8 @@ Processing happens in the following stages:
    data is written to another file (``qstrdefs.collected.h.hash``) which allows
    it to track changes across builds.
 
-4. ``qstrdefs.preprocessed.h`` adds in the QSTRs from qstrdefs*.  It
-   concatenates ``qstrdefs.collected.h`` with ``qstrdefs*.h``, then it transforms
+4. Generate an enumeration, each entry of which maps a ``MP_QSTR_Foo`` to it's corresponding index.
+   It concatenates ``qstrdefs.collected.h`` with ``qstrdefs*.h``, then it transforms
    each line from ``Q(Foo)`` to ``"Q(Foo)"`` so they pass through the preprocessor
    unchanged.  Then the preprocessor is used to deal with any conditional
    compilation in ``qstrdefs*.h``.  Then the transformation is undone back to

docs/develop/writingtests.rst

@@ -0,0 +1,70 @@
+.. _writingtests:
+
+Writing tests
+=============
+
+Tests in MicroPython are located at the path ``tests/``. The following is a listing of
+key directories and the run-tests.py runner script:
+
+.. code-block:: bash
+
+   .
+    ├── basics
+    ├── extmod
+    ├── float
+    ├── micropython
+    ├── run-tests.py
+    ...
+
+There are subfolders maintained to categorize the tests. Add a test by creating a new file in one of the
+existing folders or in a new folder. It's also possible to make custom tests outside this tests folder,
+which would be recommended for a custom port.
+
+For example, add the following code in a file ``print.py`` in the ``tests/unix/`` subdirectory:
+
+.. code-block:: python
+
+   def print_one():
+       print(1)
+
+   print_one()
+
+If you run your tests, this test should appear in the test output:
+
+.. code-block:: bash
+
+   $ cd ports/unix
+   $ make tests
+   skip  unix/extra_coverage.py
+   pass  unix/ffi_callback.py
+   pass  unix/ffi_float.py
+   pass  unix/ffi_float2.py
+   pass  unix/print.py
+   pass  unix/time.py
+   pass  unix/time2.py
+
+Tests are run by comparing the output from the test target against the output from CPython.
+So any test should use print statements to indicate test results.
+
+For tests that can't be compared to CPython (i.e. micropython-specific functionality),
+you can provide a ``.py.exp`` file which will be used as the truth for comparison.
+
+The other way to run tests, which is useful when running on targets other than the Unix port, is:
+
+.. code-block:: bash
+
+   $ cd tests
+   $ ./run-tests.py
+
+Then to run on a board:
+
+.. code-block:: bash
+
+   $ ./run-tests.py --target minimal --device /dev/ttyACM0
+
+And to run only a certain set of tests (eg a directory):
+
+.. code-block:: bash
+
+   $ ./run-tests.py -d basics
+   $ ./run-tests.py float/builtin*.py

docs/esp32/quickref.rst

@@ -249,16 +249,15 @@ ESP32 specific ADC class method reference:
 Software SPI bus
 ----------------
 
-There are two SPI drivers. One is implemented in software (bit-banging)
-and works on all pins, and is accessed via the :ref:`machine.SPI <machine.SPI>`
-class::
+Software SPI (using bit-banging) works on all pins, and is accessed via the
+:ref:`machine.SoftSPI <machine.SoftSPI>` class::
 
-    from machine import Pin, SPI
+    from machine import Pin, SoftSPI
 
-    # construct an SPI bus on the given pins
+    # construct a SoftSPI bus on the given pins
     # polarity is the idle state of SCK
     # phase=0 means sample on the first edge of SCK, phase=1 means the second
-    spi = SPI(baudrate=100000, polarity=1, phase=0, sck=Pin(0), mosi=Pin(2), miso=Pin(4))
+    spi = SoftSPI(baudrate=100000, polarity=1, phase=0, sck=Pin(0), mosi=Pin(2), miso=Pin(4))
 
     spi.init(baudrate=200000) # set the baudrate
 
@@ -298,39 +297,55 @@ mosi   13           23
 miso   12           19
 =====  ===========  ============
 
-Hardware SPI has the same methods as Software SPI above::
+Hardware SPI is accessed via the :ref:`machine.SPI <machine.SPI>` class and
+has the same methods as software SPI above::
 
     from machine import Pin, SPI
 
+    hspi = SPI(1, 10000000)
     hspi = SPI(1, 10000000, sck=Pin(14), mosi=Pin(13), miso=Pin(12))
     vspi = SPI(2, baudrate=80000000, polarity=0, phase=0, bits=8, firstbit=0, sck=Pin(18), mosi=Pin(23), miso=Pin(19))
 
+Software I2C bus
+----------------
 
-I2C bus
--------
+Software I2C (using bit-banging) works on all output-capable pins, and is
+accessed via the :ref:`machine.SoftI2C <machine.SoftI2C>` class::
 
-The I2C driver has both software and hardware implementations, and the two
-hardware peripherals have identifiers 0 and 1.  Any available output-capable
-pins can be used for SCL and SDA.  The driver is accessed via the
-:ref:`machine.I2C <machine.I2C>` class::
+    from machine import Pin, SoftI2C
 
-    from machine import Pin, I2C
+    i2c = SoftI2C(scl=Pin(5), sda=Pin(4), freq=100000)
 
-    # construct a software I2C bus
-    i2c = I2C(scl=Pin(5), sda=Pin(4), freq=100000)
+    i2c.scan()              # scan for devices
 
-    # construct a hardware I2C bus
-    i2c = I2C(0)
-    i2c = I2C(1, scl=Pin(5), sda=Pin(4), freq=400000)
-
-    i2c.scan()              # scan for slave devices
-
-    i2c.readfrom(0x3a, 4)   # read 4 bytes from slave device with address 0x3a
-    i2c.writeto(0x3a, '12') # write '12' to slave device with address 0x3a
+    i2c.readfrom(0x3a, 4)   # read 4 bytes from device with address 0x3a
+    i2c.writeto(0x3a, '12') # write '12' to device with address 0x3a
 
     buf = bytearray(10)     # create a buffer with 10 bytes
     i2c.writeto(0x3a, buf)  # write the given buffer to the slave
 
+Hardware I2C bus
+----------------
+
+There are two hardware I2C peripherals with identifiers 0 and 1.  Any available
+output-capable pins can be used for SCL and SDA but the defaults are given
+below.
+
+=====  ===========  ============
+\      I2C(0)       I2C(1)
+=====  ===========  ============
+scl    18           25
+sda    19           26
+=====  ===========  ============
+
+The driver is accessed via the :ref:`machine.I2C <machine.I2C>` class and
+has the same methods as software I2C above::
+
+    from machine import Pin, I2C
+
+    i2c = I2C(0)
+    i2c = I2C(1, scl=Pin(5), sda=Pin(4), freq=400000)
+
 Real time clock (RTC)
 ---------------------
 

docs/esp8266/quickref.rst

@@ -214,15 +214,15 @@ Software SPI bus
 ----------------
 
 There are two SPI drivers. One is implemented in software (bit-banging)
-and works on all pins, and is accessed via the :ref:`machine.SPI <machine.SPI>`
+and works on all pins, and is accessed via the :ref:`machine.SoftSPI <machine.SoftSPI>`
 class::
 
-    from machine import Pin, SPI
+    from machine import Pin, SoftSPI
 
     # construct an SPI bus on the given pins
     # polarity is the idle state of SCK
     # phase=0 means sample on the first edge of SCK, phase=1 means the second
-    spi = SPI(-1, baudrate=100000, polarity=1, phase=0, sck=Pin(0), mosi=Pin(2), miso=Pin(4))
+    spi = SoftSPI(baudrate=100000, polarity=1, phase=0, sck=Pin(0), mosi=Pin(2), miso=Pin(4))
 
     spi.init(baudrate=200000) # set the baudrate
 
@@ -258,7 +258,8 @@ I2C bus
 -------
 
 The I2C driver is implemented in software and works on all pins,
-and is accessed via the :ref:`machine.I2C <machine.I2C>` class::
+and is accessed via the :ref:`machine.I2C <machine.I2C>` class (which is an
+alias of :ref:`machine.SoftI2C <machine.SoftI2C>`)::
 
     from machine import Pin, I2C
 
@@ -363,6 +364,13 @@ For low-level driving of a NeoPixel::
     import esp
     esp.neopixel_write(pin, grb_buf, is800khz)
 
+.. Warning::
+   By default ``NeoPixel`` is configured to control the more popular *800kHz*
+   units. It is possible to use alternative timing to control other (typically
+   400kHz) devices by passing ``timing=0`` when constructing the
+   ``NeoPixel`` object.
+
+
 APA102 driver
 -------------
 

docs/library/btree.rst

@@ -118,7 +118,7 @@ Methods
 .. method:: btree.__getitem__(key)
             btree.get(key, default=None, /)
             btree.__setitem__(key, val)
-            btree.__detitem__(key)
+            btree.__delitem__(key)
             btree.__contains__(key)
 
    Standard dictionary methods.

docs/library/esp32.rst

@@ -269,3 +269,51 @@ Constants
           esp32.WAKEUP_ANY_HIGH
 
    Selects the wake level for pins.
+
+Non-Volatile Storage
+--------------------
+
+This class gives access to the Non-Volatile storage managed by ESP-IDF. The NVS is partitioned
+into namespaces and each namespace contains typed key-value pairs. The keys are strings and the
+values may be various integer types, strings, and binary blobs. The driver currently only
+supports 32-bit signed integers and blobs.
+
+.. warning::
+
+    Changes to NVS need to be committed to flash by calling the commit method. Failure
+    to call commit results in changes being lost at the next reset.
+
+.. class:: NVS(namespace)
+
+    Create an object providing access to a namespace (which is automatically created if not
+    present).
+
+.. method:: NVS.set_i32(key, value)
+
+    Sets a 32-bit signed integer value for the specified key. Remember to call *commit*!
+
+.. method:: NVS.get_i32(key)
+
+    Returns the signed integer value for the specified key. Raises an OSError if the key does not
+    exist or has a different type.
+
+.. method:: NVS.set_blob(key, value)
+
+    Sets a binary blob value for the specified key. The value passed in must support the buffer
+    protocol, e.g. bytes, bytearray, str. (Note that esp-idf distinguishes blobs and strings, this
+    method always writes a blob even if a string is passed in as value.)
+    Remember to call *commit*!
+
+.. method:: NVS.get_blob(key, buffer)
+
+    Reads the value of the blob for the specified key into the buffer, which must be a bytearray.
+    Returns the actual length read. Raises an OSError if the key does not exist, has a different
+    type, or if the buffer is too small.
+
+.. method:: NVS.erase_key(key)
+
+    Erases a key-value pair.
+
+.. method:: NVS.commit()
+
+    Commits changes made by *set_xxx* methods to flash.

docs/library/index.rst

@@ -77,7 +77,6 @@ it will fallback to loading the built-in ``ujson`` module.
    cmath.rst
    gc.rst
    math.rst
-   sys.rst
    uarray.rst
    uasyncio.rst
    ubinascii.rst
@@ -93,6 +92,7 @@ it will fallback to loading the built-in ``ujson`` module.
    usocket.rst
    ussl.rst
    ustruct.rst
+   usys.rst
    utime.rst
    uzlib.rst
    _thread.rst

docs/library/machine.I2C.rst

@@ -12,6 +12,14 @@ when created, or initialised later on.
 
 Printing the I2C object gives you information about its configuration.
 
+Both hardware and software I2C implementations exist via the
+:ref:`machine.I2C <machine.I2C>` and `machine.SoftI2C` classes.  Hardware I2C uses
+underlying hardware support of the system to perform the reads/writes and is
+usually efficient and fast but may have restrictions on which pins can be used.
+Software I2C is implemented by bit-banging and can be used on any pin but is not
+as efficient.  These classes have the same methods available and differ primarily
+in the way they are constructed.
+
 Example usage::
 
     from machine import I2C
@@ -33,22 +41,34 @@ Example usage::
 Constructors
 ------------
 
-.. class:: I2C(id=-1, *, scl, sda, freq=400000)
+.. class:: I2C(id, *, scl, sda, freq=400000)
 
    Construct and return a new I2C object using the following parameters:
 
-      - *id* identifies a particular I2C peripheral.  The default
-        value of -1 selects a software implementation of I2C which can
-        work (in most cases) with arbitrary pins for SCL and SDA.
-        If *id* is -1 then *scl* and *sda* must be specified.  Other
-        allowed values for *id* depend on the particular port/board,
-        and specifying *scl* and *sda* may or may not be required or
-        allowed in this case.
+      - *id* identifies a particular I2C peripheral.  Allowed values for
+        depend on the particular port/board
       - *scl* should be a pin object specifying the pin to use for SCL.
       - *sda* should be a pin object specifying the pin to use for SDA.
       - *freq* should be an integer which sets the maximum frequency
         for SCL.
 
+   Note that some ports/boards will have default values of *scl* and *sda*
+   that can be changed in this constructor.  Others will have fixed values
+   of *scl* and *sda* that cannot be changed.
+
+.. _machine.SoftI2C:
+.. class:: SoftI2C(scl, sda, *, freq=400000, timeout=255)
+
+   Construct a new software I2C object.  The parameters are:
+
+      - *scl* should be a pin object specifying the pin to use for SCL.
+      - *sda* should be a pin object specifying the pin to use for SDA.
+      - *freq* should be an integer which sets the maximum frequency
+        for SCL.
+      - *timeout* is the maximum time in microseconds to wait for clock
+        stretching (SCL held low by another device on the bus), after
+        which an ``OSError(ETIMEDOUT)`` exception is raised.
+
 General Methods
 ---------------
 
@@ -79,7 +99,7 @@ The following methods implement the primitive I2C master bus operations and can
 be combined to make any I2C transaction.  They are provided if you need more
 control over the bus, otherwise the standard methods (see below) can be used.
 
-These methods are available on software I2C only.
+These methods are only available on the `machine.SoftI2C` class.
 
 .. method:: I2C.start()
 

docs/library/machine.Pin.rst

@@ -33,8 +33,8 @@ Usage Model::
     # read and print the pin value
     print(p2.value())
 
-    # reconfigure pin #0 in input mode
-    p0.mode(p0.IN)
+    # reconfigure pin #0 in input mode with a pull down resistor
+    p0.init(p0.IN, p0.PULL_DOWN)
 
     # configure an irq callback
     p0.irq(lambda p:print(p))
@@ -160,25 +160,6 @@ Methods
 
    Set pin to "0" output level.
 
-.. method:: Pin.mode([mode])
-
-   Get or set the pin mode.
-   See the constructor documentation for details of the ``mode`` argument.
-
-.. method:: Pin.pull([pull])
-
-   Get or set the pin pull state.
-   See the constructor documentation for details of the ``pull`` argument.
-
-.. method:: Pin.drive([drive])
-
-   Get or set the pin drive strength.
-   See the constructor documentation for details of the ``drive`` argument.
-
-   Not all ports implement this method.
-
-   Availability: WiPy.
-
 .. method:: Pin.irq(handler=None, trigger=(Pin.IRQ_FALLING | Pin.IRQ_RISING), *, priority=1, wake=None, hard=False)
 
    Configure an interrupt handler to be called when the trigger source of the
@@ -220,6 +201,41 @@ Methods
 
    This method returns a callback object.
 
+The following methods are not part of the core Pin API and only implemented on certain ports.
+
+.. method:: Pin.low()
+
+   Set pin to "0" output level.
+
+   Availability: nrf, rp2, stm32 ports.
+
+.. method:: Pin.high()
+
+   Set pin to "1" output level.
+
+   Availability: nrf, rp2, stm32 ports.
+
+.. method:: Pin.mode([mode])
+
+   Get or set the pin mode.
+   See the constructor documentation for details of the ``mode`` argument.
+
+   Availability: cc3200, stm32 ports.
+
+.. method:: Pin.pull([pull])
+
+   Get or set the pin pull state.
+   See the constructor documentation for details of the ``pull`` argument.
+
+   Availability: cc3200, stm32 ports.
+
+.. method:: Pin.drive([drive])
+
+   Get or set the pin drive strength.
+   See the constructor documentation for details of the ``drive`` argument.
+
+   Availability: cc3200 port.
+
 Constants
 ---------
 

docs/library/machine.RTC.rst

@@ -4,7 +4,7 @@
 class RTC -- real time clock
 ============================
 
-The RTC is and independent clock that keeps track of the date
+The RTC is an independent clock that keeps track of the date
 and time.
 
 Example usage::

docs/library/machine.SPI.rst

@@ -11,21 +11,35 @@ SS (Slave Select), to select a particular device on a bus with which
 communication takes place. Management of an SS signal should happen in
 user code (via machine.Pin class).
 
+Both hardware and software SPI implementations exist via the
+:ref:`machine.SPI <machine.SPI>` and `machine.SoftSPI` classes.  Hardware SPI uses underlying
+hardware support of the system to perform the reads/writes and is usually
+efficient and fast but may have restrictions on which pins can be used.
+Software SPI is implemented by bit-banging and can be used on any pin but
+is not as efficient.  These classes have the same methods available and
+differ primarily in the way they are constructed.
+
 Constructors
 ------------
 
 .. class:: SPI(id, ...)
 
-   Construct an SPI object on the given bus, ``id``. Values of ``id`` depend
+   Construct an SPI object on the given bus, *id*. Values of *id* depend
    on a particular port and its hardware. Values 0, 1, etc. are commonly used
-   to select hardware SPI block #0, #1, etc. Value -1 can be used for
-   bitbanging (software) implementation of SPI (if supported by a port).
+   to select hardware SPI block #0, #1, etc.
 
    With no additional parameters, the SPI object is created but not
    initialised (it has the settings from the last initialisation of
    the bus, if any).  If extra arguments are given, the bus is initialised.
    See ``init`` for parameters of initialisation.
 
+.. _machine.SoftSPI:
+.. class:: SoftSPI(baudrate=500000, *, polarity=0, phase=0, bits=8, firstbit=MSB, sck=None, mosi=None, miso=None)
+
+   Construct a new software SPI object.  Additional parameters must be
+   given, usually at least *sck*, *mosi* and *miso*, and these are used
+   to initialise the bus.  See `SPI.init` for a description of the parameters.
+
 Methods
 -------
 

docs/library/machine.Signal.rst

@@ -55,7 +55,7 @@ Following is the guide when Signal vs Pin should be used:
 * Use Pin: If you implement a higher-level protocol or bus to communicate
   with more complex devices.
 
-The split between Pin and Signal come from the usecases above and the
+The split between Pin and Signal come from the use cases above and the
 architecture of MicroPython: Pin offers the lowest overhead, which may
 be important when bit-banging protocols. But Signal adds additional
 flexibility on top of Pin, at the cost of minor overhead (much smaller

docs/library/machine.Timer.rst

@@ -31,6 +31,8 @@ Constructors
 
    Construct a new timer object of the given id. Id of -1 constructs a
    virtual timer (if supported by a board).
+   
+   See ``init`` for parameters of initialisation.
 
 Methods
 -------

docs/library/machine.rst

@@ -79,7 +79,7 @@ Power related functions
    If *time_ms* is specified then this will be the maximum time in milliseconds that
    the sleep will last for.  Otherwise the sleep can last indefinitely.
 
-   With or without a timout, execution may resume at any time if there are events
+   With or without a timeout, execution may resume at any time if there are events
    that require processing.  Such events, or wake sources, should be configured before
    sleeping, like `Pin` change or `RTC` timeout.
 

docs/library/pyb.CAN.rst

@@ -49,7 +49,7 @@ Class Methods
 Methods
 -------
 
-.. method:: CAN.init(mode, extframe=False, prescaler=100, *, sjw=1, bs1=6, bs2=8, auto_restart=False)
+.. method:: CAN.init(mode, extframe=False, prescaler=100, *, sjw=1, bs1=6, bs2=8, auto_restart=False, baudrate=0, sample_point=75)
 
    Initialise the CAN bus with the given parameters:
 
@@ -67,6 +67,11 @@ Methods
      - *auto_restart* sets whether the controller will automatically try and restart
        communications after entering the bus-off state; if this is disabled then
        :meth:`~CAN.restart()` can be used to leave the bus-off state
+     - *baudrate* if a baudrate other than 0 is provided, this function will try to automatically
+       calculate a CAN bit-timing (overriding *prescaler*, *bs1* and *bs2*) that satisfies both
+       the baudrate and the desired *sample_point*.
+     - *sample_point* given in a percentage of the bit time, the *sample_point* specifies the position
+       of the last bit sample with respect to the whole bit time. The default *sample_point* is 75%.
 
    The time quanta tq is the basic unit of time for the CAN bus.  tq is the CAN
    prescaler value divided by PCLK1 (the frequency of internal peripheral bus 1);

docs/library/pyb.RTC.rst

@@ -4,7 +4,7 @@
 class RTC -- real time clock
 ============================
 
-The RTC is and independent clock that keeps track of the date
+The RTC is an independent clock that keeps track of the date
 and time.
 
 Example usage::

docs/library/uasyncio.rst

@@ -40,6 +40,10 @@ Core functions
 
     Returns the corresponding `Task` object.
 
+.. function:: current_task()
+
+    Return the `Task` object associated with the currently running task.
+
 .. function:: run(coro)
 
     Create a new task from the given coroutine and run it until it completes.
@@ -121,6 +125,9 @@ class Event
 
     Set the event.  Any tasks waiting on the event will be scheduled to run.
 
+    Note: This must be called from within a task. It is not safe to call this
+    from an IRQ, scheduler callback, or other thread. See `ThreadSafeFlag`.
+
 .. method:: Event.clear()
 
     Clear the event.
@@ -132,6 +139,29 @@ class Event
 
     This is a coroutine.
 
+class ThreadSafeFlag
+--------------------
+
+.. class:: ThreadSafeFlag()
+
+    Create a new flag which can be used to synchronise a task with code running
+    outside the asyncio loop, such as other threads, IRQs, or scheduler
+    callbacks.  Flags start in the cleared state.
+
+.. method:: ThreadSafeFlag.set()
+
+    Set the flag.  If there is a task waiting on the event, it will be scheduled
+    to run.
+
+.. method:: ThreadSafeFlag.wait()
+
+    Wait for the flag to be set.  If the flag is already set then it returns
+    immediately.
+
+    A flag may only be waited on by a single task at a time.
+
+    This is a coroutine.
+
 class Lock
 ----------
 
@@ -184,7 +214,7 @@ TCP stream connections
     This is a coroutine.
 
 .. class:: Stream()
-    
+
     This represents a TCP stream connection.  To minimise code this class implements
     both a reader and a writer, and both ``StreamReader`` and ``StreamWriter`` alias to
     this class.

docs/library/ubinascii.rst

@@ -14,13 +14,11 @@ Functions
 
 .. function:: hexlify(data, [sep])
 
-   Convert binary data to hexadecimal representation. Returns bytes string.
+   Convert the bytes in the *data* object to a hexadecimal representation.
+   Returns a bytes object.
 
-   .. admonition:: Difference to CPython
-      :class: attention
-
-      If additional argument, *sep* is supplied, it is used as a separator
-      between hexadecimal values.
+   If the additional argument *sep* is supplied it is used as a separator
+   between hexadecimal values.
 
 .. function:: unhexlify(data)
 

docs/library/ubluetooth.rst

@@ -6,8 +6,9 @@
 
 This module provides an interface to a Bluetooth controller on a board.
 Currently this supports Bluetooth Low Energy (BLE) in Central, Peripheral,
-Broadcaster, and Observer roles, and a device may operate in multiple
-roles concurrently.
+Broadcaster, and Observer roles, as well as GATT Server and Client and L2CAP
+connection-oriented-channels. A device may operate in multiple roles
+concurrently. Pairing (and bonding) is supported on some ports.
 
 This API is intended to match the low-level Bluetooth protocol and provide
 building-blocks for higher-level abstractions such as specific device types.
@@ -28,15 +29,15 @@ Constructor
 Configuration
 -------------
 
-.. method:: BLE.active([active])
+.. method:: BLE.active([active], /)
 
     Optionally changes the active state of the BLE radio, and returns the
     current state.
 
     The radio must be made active before using any other methods on this class.
 
-.. method:: BLE.config('param')
-            BLE.config(param=value, ...)
+.. method:: BLE.config('param', /)
+            BLE.config(*, param=value, ...)
 
     Get or set configuration values of the BLE interface.  To get a value the
     parameter name should be quoted as a string, and just one parameter is
@@ -45,12 +46,22 @@ Configuration
 
     Currently supported values are:
 
-    - ``'mac'``: Returns the device MAC address. If a device has a fixed address
-      (e.g. PYBD) then it will be returned. Otherwise (e.g. ESP32) a random
-      address will be generated when the BLE interface is made active.
+    - ``'mac'``: The current address in use, depending on the current address mode.
+      This returns a tuple of ``(addr_type, addr)``.
 
-      **Note:** on some ports, accessing this value requires that the interface is
-      active (so that the MAC address can be queried from the controller).
+      See :meth:`gatts_write <BLE.gap_scan>` for details about address type.
+
+      This may only be queried while the interface is currently active.
+
+    - ``'addr_mode'``: Sets the address mode. Values can be:
+
+        * 0x00 - PUBLIC - Use the controller's public address.
+        * 0x01 - RANDOM - Use a generated static address.
+        * 0x02 - RPA - Use resolvable private addresses.
+        * 0x03 - NRPA - Use non-resolvable private addresses.
+
+      By default the interface mode will use a PUBLIC address if available, otherwise
+      it will use a RANDOM address.
 
     - ``'gap_name'``: Get/set the GAP device name used by service 0x1800,
       characteristic 0x2a00.  This can be set at any time and changed multiple
@@ -60,24 +71,59 @@ Configuration
       incoming events.  This buffer is global to the entire BLE driver and so
       handles incoming data for all events, including all characteristics.
       Increasing this allows better handling of bursty incoming data (for
-      example scan results) and the ability for a central role to receive
-      larger characteristic values.
+      example scan results) and the ability to receive larger characteristic values.
+
+    - ``'mtu'``: Get/set the MTU that will be used during a ATT MTU exchange. The
+      resulting MTU will be the minimum of this and the remote device's MTU.
+      ATT MTU exchange will not happen automatically (unless the remote device initiates
+      it), and must be manually initiated with
+      :meth:`gattc_exchange_mtu<BLE.gattc_exchange_mtu>`.
+      Use the ``_IRQ_MTU_EXCHANGED`` event to discover the MTU for a given connection.
+
+    - ``'bond'``: Sets whether bonding will be enabled during pairing. When
+      enabled, pairing requests will set the "bond" flag and the keys will be stored
+      by both devices.
+
+    - ``'mitm'``: Sets whether MITM-protection is required for pairing.
+
+    - ``'io'``: Sets the I/O capabilities of this device.
+
+      Available options are::
+
+        _IO_CAPABILITY_DISPLAY_ONLY = const(0)
+        _IO_CAPABILITY_DISPLAY_YESNO = const(1)
+        _IO_CAPABILITY_KEYBOARD_ONLY = const(2)
+        _IO_CAPABILITY_NO_INPUT_OUTPUT = const(3)
+        _IO_CAPABILITY_KEYBOARD_DISPLAY = const(4)
+
+    - ``'le_secure'``: Sets whether "LE Secure" pairing is required. Default is
+      false (i.e. allow "Legacy Pairing").
 
 Event Handling
 --------------
 
-.. method:: BLE.irq(handler)
+.. method:: BLE.irq(handler, /)
 
     Registers a callback for events from the BLE stack. The *handler* takes two
     arguments, ``event`` (which will be one of the codes below) and ``data``
     (which is an event-specific tuple of values).
 
-    **Note:** the ``addr``, ``adv_data``, ``char_data``, ``notify_data``, and
-    ``uuid`` entries in the tuples are references to data managed by the
-    :mod:`ubluetooth` module (i.e. the same instance will be re-used across
-    multiple calls to the event handler). If your program wants to use this
-    data outside of the handler, then it must copy them first, e.g. by using
-    ``bytes(addr)`` or ``bluetooth.UUID(uuid)``.
+    **Note:** As an optimisation to prevent unnecessary allocations, the ``addr``,
+    ``adv_data``, ``char_data``, ``notify_data``, and ``uuid`` entries in the
+    tuples are read-only memoryview instances pointing to ubluetooth's internal
+    ringbuffer, and are only valid during the invocation of the IRQ handler
+    function.  If your program needs to save one of these values to access after
+    the IRQ handler has returned (e.g. by saving it in a class instance or global
+    variable), then it needs to take a copy of the data, either by using ``bytes()``
+    or ``bluetooth.UUID()``, like this::
+
+        connected_addr = bytes(addr)  # equivalently: adv_data, char_data, or notify_data
+        matched_uuid = bluetooth.UUID(uuid)
+
+    For example, the IRQ handler for a scan result might inspect the ``adv_data``
+    to decide if it's the correct device, and only then copy the address data to be
+    used elsewhere in the program.  And to print data from within the IRQ handler,
+    ``print(bytes(addr))`` will be needed.
 
     An event handler showing all possible events::
 
@@ -89,12 +135,12 @@ Event Handling
                 # A central has disconnected from this peripheral.
                 conn_handle, addr_type, addr = data
             elif event == _IRQ_GATTS_WRITE:
-                # A central has written to this characteristic or descriptor.
+                # A client has written to this characteristic or descriptor.
                 conn_handle, attr_handle = data
             elif event == _IRQ_GATTS_READ_REQUEST:
-                # A central has issued a read. Note: this is a hard IRQ.
-                # Return None to deny the read.
-                # Note: This event is not supported on ESP32.
+                # A client has issued a read. Note: this is only supported on STM32.
+                # Return a non-zero integer to deny the read (see below), or zero (or None)
+                # to accept the read.
                 conn_handle, attr_handle = data
             elif event == _IRQ_SCAN_RESULT:
                 # A single scan result.
@@ -143,15 +189,58 @@ Event Handling
                 # Note: Status will be zero on success, implementation-specific value otherwise.
                 conn_handle, value_handle, status = data
             elif event == _IRQ_GATTC_NOTIFY:
-                # A peripheral has sent a notify request.
+                # A server has sent a notify request.
                 conn_handle, value_handle, notify_data = data
             elif event == _IRQ_GATTC_INDICATE:
-                # A peripheral has sent an indicate request.
+                # A server has sent an indicate request.
                 conn_handle, value_handle, notify_data = data
             elif event == _IRQ_GATTS_INDICATE_DONE:
-                # A central has acknowledged the indication.
+                # A client has acknowledged the indication.
                 # Note: Status will be zero on successful acknowledgment, implementation-specific value otherwise.
                 conn_handle, value_handle, status = data
+            elif event == _IRQ_MTU_EXCHANGED:
+                # ATT MTU exchange complete (either initiated by us or the remote device).
+                conn_handle, mtu = data
+            elif event == _IRQ_L2CAP_ACCEPT:
+                # A new channel has been accepted.
+                # Return a non-zero integer to reject the connection, or zero (or None) to accept.
+                conn_handle, cid, psm, our_mtu, peer_mtu = data
+            elif event == _IRQ_L2CAP_CONNECT:
+                # A new channel is now connected (either as a result of connecting or accepting).
+                conn_handle, cid, psm, our_mtu, peer_mtu = data
+            elif event == _IRQ_L2CAP_DISCONNECT:
+                # Existing channel has disconnected (status is zero), or a connection attempt failed (non-zero status).
+                conn_handle, cid, psm, status = data
+            elif event == _IRQ_L2CAP_RECV:
+                # New data is available on the channel. Use l2cap_recvinto to read.
+                conn_handle, cid = data
+            elif event == _IRQ_L2CAP_SEND_READY:
+                # A previous l2cap_send that returned False has now completed and the channel is ready to send again.
+                # If status is non-zero, then the transmit buffer overflowed and the application should re-send the data.
+                conn_handle, cid, status = data
+            elif event == _IRQ_CONNECTION_UPDATE:
+                # The remote device has updated connection parameters.
+                conn_handle, conn_interval, conn_latency, supervision_timeout, status = data
+            elif event == _IRQ_ENCRYPTION_UPDATE:
+                # The encryption state has changed (likely as a result of pairing or bonding).
+                conn_handle, encrypted, authenticated, bonded, key_size = data
+            elif event == _IRQ_GET_SECRET:
+                # Return a stored secret.
+                # If key is None, return the index'th value of this sec_type.
+                # Otherwise return the corresponding value for this sec_type and key.
+                sec_type, index, key = data
+                return value
+            elif event == _IRQ_SET_SECRET:
+                # Save a secret to the store for this sec_type and key.
+                sec_type, key, value = data
+                return True
+            elif event == _IRQ_PASSKEY_ACTION:
+                # Respond to a passkey request during pairing.
+                # See gap_passkey() for details.
+                # action will be an action that is compatible with the configured "io" config.
+                # passkey will be non-zero if action is "numeric comparison".
+                conn_handle, action, passkey = data
+
 
 The event codes are::
 
@@ -176,6 +265,32 @@ The event codes are::
     _IRQ_GATTC_NOTIFY = const(18)
     _IRQ_GATTC_INDICATE = const(19)
     _IRQ_GATTS_INDICATE_DONE = const(20)
+    _IRQ_MTU_EXCHANGED = const(21)
+    _IRQ_L2CAP_ACCEPT = const(22)
+    _IRQ_L2CAP_CONNECT = const(23)
+    _IRQ_L2CAP_DISCONNECT = const(24)
+    _IRQ_L2CAP_RECV = const(25)
+    _IRQ_L2CAP_SEND_READY = const(26)
+    _IRQ_CONNECTION_UPDATE = const(27)
+    _IRQ_ENCRYPTION_UPDATE = const(28)
+    _IRQ_GET_SECRET = const(29)
+    _IRQ_SET_SECRET = const(30)
+
+For the ``_IRQ_GATTS_READ_REQUEST`` event, the available return codes are::
+
+    _GATTS_NO_ERROR = const(0x00)
+    _GATTS_ERROR_READ_NOT_PERMITTED = const(0x02)
+    _GATTS_ERROR_WRITE_NOT_PERMITTED = const(0x03)
+    _GATTS_ERROR_INSUFFICIENT_AUTHENTICATION = const(0x05)
+    _GATTS_ERROR_INSUFFICIENT_AUTHORIZATION = const(0x08)
+    _GATTS_ERROR_INSUFFICIENT_ENCRYPTION = const(0x0f)
+
+For the ``_IRQ_PASSKEY_ACTION`` event, the available actions are::
+
+    _PASSKEY_ACTION_NONE = const(0)
+    _PASSKEY_ACTION_INPUT = const(2)
+    _PASSKEY_ACTION_DISPLAY = const(3)
+    _PASSKEY_ACTION_NUMERIC_COMPARISON = const(4)
 
 In order to save space in the firmware, these constants are not included on the
 :mod:`ubluetooth` module. Add the ones that you need from the list above to your
@@ -185,7 +300,7 @@ program.
 Broadcaster Role (Advertiser)
 -----------------------------
 
-.. method:: BLE.gap_advertise(interval_us, adv_data=None, resp_data=None, connectable=True)
+.. method:: BLE.gap_advertise(interval_us, adv_data=None, *, resp_data=None, connectable=True)
 
     Starts advertising at the specified interval (in **micro**\ seconds). This
     interval will be rounded down to the nearest 625us. To stop advertising, set
@@ -204,7 +319,7 @@ Broadcaster Role (Advertiser)
 Observer Role (Scanner)
 -----------------------
 
-.. method:: BLE.gap_scan(duration_ms, [interval_us], [window_us], [active])
+.. method:: BLE.gap_scan(duration_ms, interval_us=1280000, window_us=11250, active=False, /)
 
     Run a scan operation lasting for the specified duration (in **milli**\ seconds).
 
@@ -219,8 +334,13 @@ Observer Role (Scanner)
     (background scanning).
 
     For each scan result the ``_IRQ_SCAN_RESULT`` event will be raised, with event
-    data ``(addr_type, addr, adv_type, rssi, adv_data)``.  ``adv_type`` values correspond
-    to the Bluetooth Specification:
+    data ``(addr_type, addr, adv_type, rssi, adv_data)``.
+
+    ``addr_type`` values indicate public or random addresses:
+        * 0x00 - PUBLIC
+        * 0x01 - RANDOM (either static, RPA, or NRPA, the type is encoded in the address itself)
+
+    ``adv_type`` values correspond to the Bluetooth Specification:
 
         * 0x00 - ADV_IND - connectable and scannable undirected advertising
         * 0x01 - ADV_DIRECT_IND - connectable directed advertising
@@ -229,33 +349,79 @@ Observer Role (Scanner)
         * 0x04 - SCAN_RSP - scan response
 
     ``active`` can be set ``True`` if you want to receive scan responses in the results.
-    
+
     When scanning is stopped (either due to the duration finishing or when
     explicitly stopped), the ``_IRQ_SCAN_DONE`` event will be raised.
 
 
-Peripheral Role (GATT Server)
------------------------------
+Central Role
+------------
 
-A BLE peripheral has a set of registered services. Each service may contain
+A central device can connect to peripherals that it has discovered using the observer role (see :meth:`gap_scan<BLE.gap_scan>`) or with a known address.
+
+.. method:: BLE.gap_connect(addr_type, addr, scan_duration_ms=2000, /)
+
+    Connect to a peripheral.
+
+    See :meth:`gap_scan <BLE.gap_scan>` for details about address types.
+
+    On success, the ``_IRQ_PERIPHERAL_CONNECT`` event will be raised.
+
+
+Peripheral Role
+---------------
+
+A peripheral device is expected to send connectable advertisements (see
+:meth:`gap_advertise<BLE.gap_advertise>`). It will usually be acting as a GATT
+server, having first registered services and characteristics using
+:meth:`gatts_register_services<BLE.gatts_register_services>`.
+
+When a central connects, the ``_IRQ_CENTRAL_CONNECT`` event will be raised.
+
+
+Central & Peripheral Roles
+--------------------------
+
+.. method:: BLE.gap_disconnect(conn_handle, /)
+
+    Disconnect the specified connection handle. This can either be a
+    central that has connected to this device (if acting as a peripheral)
+    or a peripheral that was previously connected to by this device (if acting
+    as a central).
+
+    On success, the ``_IRQ_PERIPHERAL_DISCONNECT`` or ``_IRQ_CENTRAL_DISCONNECT``
+    event will be raised.
+
+    Returns ``False`` if the connection handle wasn't connected, and ``True``
+    otherwise.
+
+
+GATT Server
+-----------
+
+A GATT server has a set of registered services. Each service may contain
 characteristics, which each have a value. Characteristics can also contain
 descriptors, which themselves have values.
 
 These values are stored locally, and are accessed by their "value handle" which
 is generated during service registration. They can also be read from or written
-to by a remote central device. Additionally, a peripheral can "notify" a
-characteristic to a connected central via a connection handle.
+to by a remote client device. Additionally, a server can "notify" a
+characteristic to a connected client via a connection handle.
+
+A device in either central or peripheral roles may function as a GATT server,
+however in most cases it will be more common for a peripheral device to act
+as the server.
 
 Characteristics and descriptors have a default maximum size of 20 bytes.
-Anything written to them by a central will be truncated to this length. However,
+Anything written to them by a client will be truncated to this length. However,
 any local write will increase the maximum size, so if you want to allow larger
-writes from a central to a given characteristic, use
+writes from a client to a given characteristic, use
 :meth:`gatts_write<BLE.gatts_write>` after registration. e.g.
 ``gatts_write(char_handle, bytes(100))``.
 
-.. method:: BLE.gatts_register_services(services_definition)
+.. method:: BLE.gatts_register_services(services_definition, /)
 
-    Configures the peripheral with the specified services, replacing any
+    Configures the server with the specified services, replacing any
     existing services.
 
     *services_definition* is a list of **services**, where each **service** is a
@@ -267,9 +433,9 @@ writes from a central to a given characteristic, use
     Each **descriptor** is a two-element tuple containing a UUID and a **flags**
     value.
 
-    The **flags** are a bitwise-OR combination of the
-    :data:`ubluetooth.FLAG_READ`, :data:`ubluetooth.FLAG_WRITE` and
-    :data:`ubluetooth.FLAG_NOTIFY` values defined below.
+    The **flags** are a bitwise-OR combination of the flags defined below. These
+    set both the behaviour of the characteristic (or descriptor) as well as the
+    security and privacy requirements.
 
     The return value is a list (one element per service) of tuples (each element
     is a value handle). Characteristics and descriptor handles are flattened
@@ -293,28 +459,49 @@ writes from a central to a given characteristic, use
 
     **Note:** Advertising must be stopped before registering services.
 
-.. method:: BLE.gatts_read(value_handle)
+    Available flags for characteristics and descriptors are::
+
+        from micropython import const
+        _FLAG_BROADCAST = const(0x0001)
+        _FLAG_READ = const(0x0002)
+        _FLAG_WRITE_NO_RESPONSE = const(0x0004)
+        _FLAG_WRITE = const(0x0008)
+        _FLAG_NOTIFY = const(0x0010)
+        _FLAG_INDICATE = const(0x0020)
+        _FLAG_AUTHENTICATED_SIGNED_WRITE = const(0x0040)
+
+        _FLAG_AUX_WRITE = const(0x0100)
+        _FLAG_READ_ENCRYPTED = const(0x0200)
+        _FLAG_READ_AUTHENTICATED = const(0x0400)
+        _FLAG_READ_AUTHORIZED = const(0x0800)
+        _FLAG_WRITE_ENCRYPTED = const(0x1000)
+        _FLAG_WRITE_AUTHENTICATED = const(0x2000)
+        _FLAG_WRITE_AUTHORIZED = const(0x4000)
+
+    As for the IRQs above, any required constants should be added to your Python code.
+
+.. method:: BLE.gatts_read(value_handle, /)
 
     Reads the local value for this handle (which has either been written by
-    :meth:`gatts_write <BLE.gatts_write>` or by a remote central).
+    :meth:`gatts_write <BLE.gatts_write>` or by a remote client).
 
-.. method:: BLE.gatts_write(value_handle, data)
+.. method:: BLE.gatts_write(value_handle, data, /)
 
-    Writes the local value for this handle, which can be read by a central.
+    Writes the local value for this handle, which can be read by a client.
 
-.. method:: BLE.gatts_notify(conn_handle, value_handle, [data])
+.. method:: BLE.gatts_notify(conn_handle, value_handle, data=None, /)
 
-    Sends a notification request to a connected central.
+    Sends a notification request to a connected client.
 
-    If *data* is specified, then that value is sent to the central as part of
+    If *data* is not ``None``, then that value is sent to the client as part of
     the notification. The local value will not be modified.
 
-    Otherwise, if *data* is not specified, then the current local value (as
+    Otherwise, if *data* is ``None``, then the current local value (as
     set with :meth:`gatts_write <BLE.gatts_write>`) will be sent.
 
-.. method:: BLE.gatts_indicate(conn_handle, value_handle)
+.. method:: BLE.gatts_indicate(conn_handle, value_handle, /)
 
-    Sends an indication request to a connected central.
+    Sends an indication request to a connected client.
 
     **Note:** This does not currently support sending a custom value, it will
     always send the current local value (as set with :meth:`gatts_write
@@ -334,37 +521,28 @@ writes from a central to a given characteristic, use
     be cleared after reading. This feature is useful when implementing something
     like the Nordic UART Service.
 
+GATT Client
+-----------
 
-Central Role (GATT Client)
---------------------------
+A GATT client can discover and read/write characteristics on a remote GATT server.
 
-.. method:: BLE.gap_connect(addr_type, addr, scan_duration_ms=2000, /)
+It is more common for a central role device to act as the GATT client, however
+it's also possible for a peripheral to act as a client in order to discover
+information about the central that has connected to it (e.g. to read the
+device name from the device information service).
 
-    Connect to a peripheral.
+.. method:: BLE.gattc_discover_services(conn_handle, uuid=None, /)
 
-    On success, the ``_IRQ_PERIPHERAL_CONNECT`` event will be raised.
-
-.. method:: BLE.gap_disconnect(conn_handle)
-
-    Disconnect the specified connection handle.
-
-    On success, the ``_IRQ_PERIPHERAL_DISCONNECT`` event will be raised.
-
-    Returns ``False`` if the connection handle wasn't connected, and ``True``
-    otherwise.
-
-.. method:: BLE.gattc_discover_services(conn_handle, [uuid])
-
-    Query a connected peripheral for its services.
+    Query a connected server for its services.
 
     Optionally specify a service *uuid* to query for that service only.
 
     For each service discovered, the ``_IRQ_GATTC_SERVICE_RESULT`` event will
     be raised, followed by ``_IRQ_GATTC_SERVICE_DONE`` on completion.
 
-.. method:: BLE.gattc_discover_characteristics(conn_handle, start_handle, end_handle, [uuid])
+.. method:: BLE.gattc_discover_characteristics(conn_handle, start_handle, end_handle, uuid=None, /)
 
-    Query a connected peripheral for characteristics in the specified range.
+    Query a connected server for characteristics in the specified range.
 
     Optionally specify a characteristic *uuid* to query for that
     characteristic only.
@@ -375,16 +553,16 @@ Central Role (GATT Client)
     For each characteristic discovered, the ``_IRQ_GATTC_CHARACTERISTIC_RESULT``
     event will be raised, followed by ``_IRQ_GATTC_CHARACTERISTIC_DONE`` on completion.
 
-.. method:: BLE.gattc_discover_descriptors(conn_handle, start_handle, end_handle)
+.. method:: BLE.gattc_discover_descriptors(conn_handle, start_handle, end_handle, /)
 
-    Query a connected peripheral for descriptors in the specified range.
+    Query a connected server for descriptors in the specified range.
 
     For each descriptor discovered, the ``_IRQ_GATTC_DESCRIPTOR_RESULT`` event
     will be raised, followed by ``_IRQ_GATTC_DESCRIPTOR_DONE`` on completion.
 
-.. method:: BLE.gattc_read(conn_handle, value_handle)
+.. method:: BLE.gattc_read(conn_handle, value_handle, /)
 
-    Issue a remote read to a connected peripheral for the specified
+    Issue a remote read to a connected server for the specified
     characteristic or descriptor handle.
 
     When a value is available, the ``_IRQ_GATTC_READ_RESULT`` event will be
@@ -392,22 +570,158 @@ Central Role (GATT Client)
 
 .. method:: BLE.gattc_write(conn_handle, value_handle, data, mode=0, /)
 
-    Issue a remote write to a connected peripheral for the specified
+    Issue a remote write to a connected server for the specified
     characteristic or descriptor handle.
 
     The argument *mode* specifies the write behaviour, with the currently
     supported values being:
 
         * ``mode=0`` (default) is a write-without-response: the write will
-          be sent to the remote peripheral but no confirmation will be
+          be sent to the remote server but no confirmation will be
           returned, and no event will be raised.
-        * ``mode=1`` is a write-with-response: the remote peripheral is
+        * ``mode=1`` is a write-with-response: the remote server is
           requested to send a response/acknowledgement that it received the
           data.
 
-    If a response is received from the remote peripheral the
+    If a response is received from the remote server the
     ``_IRQ_GATTC_WRITE_DONE`` event will be raised.
 
+.. method:: BLE.gattc_exchange_mtu(conn_handle, /)
+
+    Initiate MTU exchange with a connected server, using the preferred MTU
+    set using ``BLE.config(mtu=value)``.
+
+    The ``_IRQ_MTU_EXCHANGED`` event will be raised when MTU exchange
+    completes.
+
+    **Note:** MTU exchange is typically initiated by the central. When using
+    the BlueKitchen stack in the central role, it does not support a remote
+    peripheral initiating the MTU exchange. NimBLE works for both roles.
+
+
+L2CAP connection-oriented-channels
+----------------------------------
+
+    This feature allows for socket-like data exchange between two BLE devices.
+    Once the devices are connected via GAP, either device can listen for the
+    other to connect on a numeric PSM (Protocol/Service Multiplexer).
+
+    **Note:** This is currently only supported when using the NimBLE stack on
+    STM32 and Unix (not ESP32). Only one L2CAP channel may be active at a given
+    time (i.e. you cannot connect while listening).
+
+    Active L2CAP channels are identified by the connection handle that they were
+    established on and a CID (channel ID).
+
+    Connection-oriented channels have built-in credit-based flow control. Unlike
+    ATT, where devices negotiate a shared MTU, both the listening and connecting
+    devices each set an independent MTU which limits the maximum amount of
+    outstanding data that the remote device can send before it is fully consumed
+    in :meth:`l2cap_recvinto <BLE.l2cap_recvinto>`.
+
+.. method:: BLE.l2cap_listen(psm, mtu, /)
+
+    Start listening for incoming L2CAP channel requests on the specified *psm*
+    with the local MTU set to *mtu*.
+
+    When a remote device initiates a connection, the ``_IRQ_L2CAP_ACCEPT``
+    event will be raised, which gives the listening server a chance to reject
+    the incoming connection (by returning a non-zero integer).
+
+    Once the connection is accepted, the ``_IRQ_L2CAP_CONNECT`` event will be
+    raised, allowing the server to obtain the channel id (CID) and the local and
+    remote MTU.
+
+    **Note:** It is not currently possible to stop listening.
+
+.. method:: BLE.l2cap_connect(conn_handle, psm, mtu, /)
+
+    Connect to a listening peer on the specified *psm* with local MTU set to *mtu*.
+
+    On successful connection, the the ``_IRQ_L2CAP_CONNECT`` event will be
+    raised, allowing the client to obtain the CID and the local and remote (peer) MTU.
+
+    An unsuccessful connection will raise the ``_IRQ_L2CAP_DISCONNECT`` event
+    with a non-zero status.
+
+.. method:: BLE.l2cap_disconnect(conn_handle, cid, /)
+
+    Disconnect an active L2CAP channel with the specified *conn_handle* and
+    *cid*.
+
+.. method:: BLE.l2cap_send(conn_handle, cid, buf, /)
+
+    Send the specified *buf* (which must support the buffer protocol) on the
+    L2CAP channel identified by *conn_handle* and *cid*.
+
+    The specified buffer cannot be larger than the remote (peer) MTU, and no
+    more than twice the size of the local MTU.
+
+    This will return ``False`` if the channel is now "stalled", which means that
+    :meth:`l2cap_send <BLE.l2cap_send>` must not be called again until the
+    ``_IRQ_L2CAP_SEND_READY`` event is received (which will happen when the
+    remote device grants more credits, typically after it has received and
+    processed the data).
+
+.. method:: BLE.l2cap_recvinto(conn_handle, cid, buf, /)
+
+    Receive data from the specified *conn_handle* and *cid* into the provided
+    *buf* (which must support the buffer protocol, e.g. bytearray or
+    memoryview).
+
+    Returns the number of bytes read from the channel.
+
+    If *buf* is None, then returns the number of bytes available.
+
+    **Note:** After receiving the ``_IRQ_L2CAP_RECV`` event, the application should
+    continue calling :meth:`l2cap_recvinto <BLE.l2cap_recvinto>` until no more
+    bytes are available in the receive buffer (typically up to the size of the
+    remote (peer) MTU).
+
+    Until the receive buffer is empty, the remote device will not be granted
+    more channel credits and will be unable to send any more data.
+
+
+Pairing and bonding
+-------------------
+
+    Pairing allows a connection to be encrypted and authenticated via exchange
+    of secrets (with optional MITM protection via passkey authentication).
+
+    Bonding is the process of storing those secrets into non-volatile storage.
+    When bonded, a device is able to resolve a resolvable private address (RPA)
+    from another device based on the stored identity resolving key (IRK).
+    To support bonding, an application must implement the ``_IRQ_GET_SECRET``
+    and ``_IRQ_SET_SECRET`` events.
+
+    **Note:** This is currently only supported when using the NimBLE stack on
+    STM32 and Unix (not ESP32).
+
+.. method:: BLE.gap_pair(conn_handle, /)
+
+    Initiate pairing with the remote device.
+
+    Before calling this, ensure that the ``io``, ``mitm``, ``le_secure``, and
+    ``bond`` configuration options are set (via :meth:`config<BLE.config>`).
+
+    On successful pairing, the ``_IRQ_ENCRYPTION_UPDATE`` event will be raised.
+
+.. method:: BLE.gap_passkey(conn_handle, action, passkey, /)
+
+    Respond to a ``_IRQ_PASSKEY_ACTION`` event for the specified *conn_handle*
+    and *action*.
+
+    The *passkey* is a numeric value and will depend on on the
+    *action* (which will depend on what I/O capability has been set):
+
+        * When the *action* is ``_PASSKEY_ACTION_INPUT``, then the application should
+          prompt the user to enter the passkey that is shown on the remote device.
+        * When the *action* is ``_PASSKEY_ACTION_DISPLAY``, then the application should
+          generate a random 6-digit passkey and show it to the user.
+        * When the *action* is ``_PASSKEY_ACTION_NUMERIC_COMPARISON``, then the application
+          should show the passkey that was provided in the ``_IRQ_PASSKEY_ACTION`` event
+          and then respond with either ``0`` (cancel pairing), or ``1`` (accept pairing).
+
 
 class UUID
 ----------
@@ -416,7 +730,7 @@ class UUID
 Constructor
 -----------
 
-.. class:: UUID(value)
+.. class:: UUID(value, /)
 
     Creates a UUID instance with the specified **value**.
 
@@ -424,11 +738,3 @@ Constructor
 
     - A 16-bit integer. e.g. ``0x2908``.
     - A 128-bit UUID string. e.g. ``'6E400001-B5A3-F393-E0A9-E50E24DCCA9E'``.
-
-
-Constants
----------
-
-.. data:: ubluetooth.FLAG_READ
-          ubluetooth.FLAG_WRITE
-          ubluetooth.FLAG_NOTIFY

docs/library/ussl.rst

@@ -13,16 +13,23 @@ facilities for network sockets, both client-side and server-side.
 Functions
 ---------
 
-.. function:: ussl.wrap_socket(sock, server_side=False, keyfile=None, certfile=None, cert_reqs=CERT_NONE, ca_certs=None)
-
+.. function:: ussl.wrap_socket(sock, server_side=False, keyfile=None, certfile=None, cert_reqs=CERT_NONE, ca_certs=None, do_handshake=True) 
    Takes a `stream` *sock* (usually usocket.socket instance of ``SOCK_STREAM`` type),
    and returns an instance of ssl.SSLSocket, which wraps the underlying stream in
    an SSL context. Returned object has the usual `stream` interface methods like
-   ``read()``, ``write()``, etc. In MicroPython, the returned object does not expose
-   socket interface and methods like ``recv()``, ``send()``. In particular, a
-   server-side SSL socket should be created from a normal socket returned from
+   ``read()``, ``write()``, etc.
+   A server-side SSL socket should be created from a normal socket returned from
    :meth:`~usocket.socket.accept()` on a non-SSL listening server socket.
 
+   - *do_handshake* determines whether the handshake is done as part of the ``wrap_socket``
+     or whether it is deferred to be done as part of the initial reads or writes
+     (there is no ``do_handshake`` method as in CPython).
+     For blocking sockets doing the handshake immediately is standard. For non-blocking
+     sockets (i.e. when the *sock* passed into ``wrap_socket`` is in non-blocking mode)
+     the handshake should generally be deferred because otherwise ``wrap_socket`` blocks
+     until it completes. Note that in AXTLS the handshake can be deferred until the first
+     read or write but it then blocks until completion.
+
    Depending on the underlying module implementation in a particular
    :term:`MicroPython port`, some or all keyword arguments above may be not supported.
 
@@ -31,6 +38,11 @@ Functions
    Some implementations of ``ussl`` module do NOT validate server certificates,
    which makes an SSL connection established prone to man-in-the-middle attacks.
 
+   CPython's ``wrap_socket`` returns an ``SSLSocket`` object which has methods typical
+   for sockets, such as ``send``, ``recv``, etc. MicroPython's ``wrap_socket``
+   returns an object more similar to CPython's ``SSLObject`` which does not have
+   these socket methods.
+
 Exceptions
 ----------
 

docs/library/usys.rst

@@ -1,7 +1,7 @@
-:mod:`sys` -- system specific functions
-=======================================
+:mod:`usys` -- system specific functions
+========================================
 
-.. module:: sys
+.. module:: usys
    :synopsis: system specific functions
 
 |see_cpython_module| :mod:`python:sys`.
@@ -28,10 +28,10 @@ Functions
       This function is a MicroPython extension intended to provide similar
       functionality to the :mod:`atexit` module in CPython.
 
-.. function:: print_exception(exc, file=sys.stdout, /)
+.. function:: print_exception(exc, file=usys.stdout, /)
 
    Print exception with a traceback to a file-like object *file* (or
-   `sys.stdout` by default).
+   `usys.stdout` by default).
 
    .. admonition:: Difference to CPython
       :class: attention
@@ -84,7 +84,7 @@ Constants
    value directly, but instead count number of bits in it::
 
     bits = 0
-    v = sys.maxsize
+    v = usys.maxsize
     while v:
         bits += 1
         v >>= 1
@@ -113,7 +113,7 @@ Constants
    is an identifier of a board, e.g. ``"pyboard"`` for the original MicroPython
    reference board. It thus can be used to distinguish one board from another.
    If you need to check whether your program runs on MicroPython (vs other
-   Python implementation), use `sys.implementation` instead.
+   Python implementation), use `usys.implementation` instead.
 
 .. data:: stderr
 

docs/library/utime.rst

@@ -36,11 +36,17 @@ behave not as expected.
 Functions
 ---------
 
-.. function:: localtime([secs])
+.. function:: gmtime([secs])
+              localtime([secs])
 
-   Convert a time expressed in seconds since the Epoch (see above) into an 8-tuple which
-   contains: (year, month, mday, hour, minute, second, weekday, yearday)
-   If secs is not provided or None, then the current time from the RTC is used.
+   Convert the time *secs* expressed in seconds since the Epoch (see above) into an
+   8-tuple which contains: ``(year, month, mday, hour, minute, second, weekday, yearday)``
+   If *secs* is not provided or None, then the current time from the RTC is used.
+
+   The `gmtime()` function returns a date-time tuple in UTC, and `localtime()` returns a
+   date-time tuple in local time.
+
+   The format of the entries in the 8-tuple are:
 
    * year includes the century (for example 2014).
    * month   is 1-12
@@ -210,8 +216,9 @@ Functions
    function returns number of seconds since a port-specific reference point in time (for
    embedded boards without a battery-backed RTC, usually since power up or reset). If you
    want to develop portable MicroPython application, you should not rely on this function
-   to provide higher than second precision. If you need higher precision, use
-   `ticks_ms()` and `ticks_us()` functions, if you need calendar time,
+   to provide higher than second precision.  If you need higher precision, absolute
+   timestamps, use `time_ns()`.  If relative times are acceptable then use the
+   `ticks_ms()` and `ticks_us()` functions.  If you need calendar time, `gmtime()` or
    `localtime()` without an argument is a better choice.
 
    .. admonition:: Difference to CPython
@@ -227,3 +234,8 @@ Functions
       hardware also lacks battery-powered RTC, so returns number of seconds
       since last power-up or from other relative, hardware-specific point
       (e.g. reset).
+
+.. function:: time_ns()
+
+    Similar to `time()` but returns nanoseconds since the Epoch, as an integer (usually
+    a big integer, so will allocate on the heap).

docs/reference/glossary.rst

@@ -177,7 +177,7 @@ Glossary
         typically accessible on a host PC via USB.
 
     stream
-        Also known as a "file-like object". An Python object which provides
+        Also known as a "file-like object". A Python object which provides
         sequential read-write access to the underlying data. A stream object
         implements a corresponding interface, which consists of methods like
         ``read()``, ``write()``, ``readinto()``, ``seek()``, ``flush()``,

docs/reference/packages.rst

@@ -188,7 +188,7 @@ Few notes:
 1. Step 5 in the sequence above assumes that the distribution package
    is available from PyPI. If that is not the case, you would need
    to copy Python source files manually to ``modules/`` subdirectory
-   of the port port directory. (Note that upip does not support
+   of the port directory. (Note that upip does not support
    installing from e.g. version control repositories).
 2. The firmware for baremetal devices usually has size restrictions,
    so adding too many frozen modules may overflow it. Usually, you

docs/reference/pyboard.py.rst

@@ -48,7 +48,9 @@ Running ``pyboard.py --help`` gives the following output:
                             available
       --follow              follow the output after running the scripts
                             [default if no scripts given]
-      -f, --filesystem      perform a filesystem action
+      -f, --filesystem      perform a filesystem action: cp local :device | cp
+                            :device local | cat path | ls [path] | rm path | mkdir
+                            path | rmdir path
 
 Running a command on the device
 -------------------------------

docs/reference/repl.rst

@@ -196,17 +196,105 @@ So you can use the underscore to save the result in a variable. For example:
     15
     >>>
 
-Raw mode
---------
+Raw mode and raw-paste mode
+---------------------------
 
-Raw mode is not something that a person would normally use. It is intended for
-programmatic use. It essentially behaves like paste mode with echo turned off.
+Raw mode (also called raw REPL) is not something that a person would normally use.
+It is intended for programmatic use and essentially behaves like paste mode with
+echo turned off, and with optional flow control.
 
 Raw mode is entered using Ctrl-A. You then send your python code, followed by
 a Ctrl-D. The Ctrl-D will be acknowledged by 'OK' and then the python code will
 be compiled and executed. Any output (or errors) will be sent back. Entering
 Ctrl-B will leave raw mode and return the the regular (aka friendly) REPL.
 
-The ``tools/pyboard.py`` program uses the raw REPL to execute python files on the
-MicroPython board.
+Raw-paste mode is an additional mode within the raw REPL that includes flow control,
+and which compiles code as it receives it. This makes it more robust for high-speed
+transfer of code into the device, and it also uses less RAM when receiving because
+it does not need to store a verbatim copy of the code before compiling (unlike
+standard raw mode).
 
+Raw-paste mode uses the following protocol:
+
+#. Enter raw REPL as usual via ctrl-A.
+
+#. Write 3 bytes: ``b"\x05A\x01"`` (ie ctrl-E then "A" then ctrl-A).
+
+#. Read 2 bytes to determine if the device entered raw-paste mode:
+
+   * If the result is ``b"R\x00"`` then the device understands the command but
+     doesn't support raw paste.
+
+   * If the result is ``b"R\x01"`` then the device does support raw paste and
+     has entered this mode.
+
+   * Otherwise the result should be ``b"ra"`` and the device doesn't support raw
+     paste and the string ``b"w REPL; CTRL-B to exit\r\n>"`` should be read and
+     discarded.
+
+#. If the device is in raw-paste mode then continue, otherwise fallback to
+   standard raw mode.
+
+#. Read 2 bytes, this is the flow control window-size-increment (in bytes)
+   stored as a 16-bit unsigned little endian integer.  The initial value for the
+   remaining-window-size variable should be set to this number.
+
+#. Write out the code to the device:
+
+   * While there are bytes to send, write up to the remaining-window-size worth
+     of bytes, and decrease the remaining-window-size by the number of bytes
+     written.
+
+   * If the remaining-window-size is 0, or there is a byte waiting to read, read
+     1 byte.  If this byte is ``b"\x01"`` then increase the remaining-window-size
+     by the window-size-increment from step 5.  If this byte is ``b"\x04"`` then
+     the device wants to end the data reception, and ``b"\x04"`` should be
+     written to the device and no more code sent after that.  (Note: if there is
+     a byte waiting to be read from the device then it does not need to be read
+     and acted upon immediately, the device will continue to consume incoming
+     bytes as long as reamining-window-size is greater than 0.)
+
+#. When all code has been written to the device, write ``b"\x04"`` to indicate
+   end-of-data.
+
+#. Read from the device until ``b"\x04"`` is received.  At this point the device
+   has received and compiled all of the code that was sent and is executing it.
+
+#. The device outputs any characters produced by the executing code.  When (if)
+   the code finishes ``b"\x04"`` will be output, followed by any exception that
+   was uncaught, followed again by ``b"\x04"``.  It then goes back to the
+   standard raw REPL and outputs ``b">"``.
+
+For example, starting at a new line at the normal (friendly) REPL, if you write::
+
+    b"\x01\x05A\x01print(123)\x04"
+
+Then the device will respond with something like::
+
+    b"\r\nraw REPL; CTRL-B to exit\r\n>R\x01\x80\x00\x01\x04123\r\n\x04\x04>"
+
+Broken down over time this looks like::
+
+    # Step 1: enter raw REPL
+    write: b"\x01"
+    read: b"\r\nraw REPL; CTRL-B to exit\r\n>"
+
+    # Step 2-5: enter raw-paste mode
+    write: b"\x05A\x01"
+    read: b"R\x01\x80\x00\x01"
+
+    # Step 6-8: write out code
+    write: b"print(123)\x04"
+    read: b"\x04"
+
+    # Step 9: code executes and result is read
+    read: b"123\r\n\x04\x04>"
+
+In this case the flow control window-size-increment is 128 and there are two
+windows worth of data immediately available at the start, one from the initial
+window-size-increment value and one from the explicit ``b"\x01"`` value that
+is sent.  So this means up to 256 bytes can be written to begin with before
+waiting or checking for more incoming flow-control characters.
+
+The ``tools/pyboard.py`` program uses the raw REPL, including raw-paste mode, to
+execute Python code on a MicroPython-enabled board.

drivers/cyw43/cywbt.c

@@ -31,13 +31,19 @@
 #include "py/mphal.h"
 #include "pin_static_af.h"
 #include "uart.h"
-#include "extmod/modbluetooth_hci.h"
+#include "extmod/mpbthci.h"
 
 #if MICROPY_PY_NETWORK_CYW43
 
 extern const char fw_4343WA1_7_45_98_50_start;
 #define CYWBT_FW_ADDR (&fw_4343WA1_7_45_98_50_start + 749 * 512 + 29 * 256)
 
+// Provided by the port.
+extern pyb_uart_obj_t mp_bluetooth_hci_uart_obj;
+
+// Provided by the port, and also possibly shared with the stack.
+extern uint8_t mp_bluetooth_hci_cmd_buf[4 + 256];
+
 /******************************************************************************/
 // CYW BT HCI low-level driver
 
@@ -168,10 +174,6 @@ int mp_bluetooth_hci_controller_init(void) {
     mp_hal_pin_config(pyb_pin_WL_GPIO_4, MP_HAL_PIN_MODE_OUTPUT, MP_HAL_PIN_PULL_NONE, 0); // RF-switch power
     mp_hal_pin_high(pyb_pin_WL_GPIO_4); // Turn the RF-switch on
 
-    return 0;
-}
-
-int mp_bluetooth_hci_controller_activate(void) {
     uint8_t buf[256];
 
     mp_hal_pin_low(pyb_pin_BT_REG_ON);
@@ -219,7 +221,7 @@ int mp_bluetooth_hci_controller_activate(void) {
     return 0;
 }
 
-int mp_bluetooth_hci_controller_deactivate(void) {
+int mp_bluetooth_hci_controller_deinit(void) {
     mp_hal_pin_low(pyb_pin_BT_REG_ON);
 
     return 0;

drivers/memory/spiflash.c

@@ -287,6 +287,8 @@ int mp_spiflash_write(mp_spiflash_t *self, uint32_t addr, size_t len, const uint
 /******************************************************************************/
 // Interface functions that use the cache
 
+#if MICROPY_HW_SPIFLASH_ENABLE_CACHE
+
 void mp_spiflash_cached_read(mp_spiflash_t *self, uint32_t addr, size_t len, uint8_t *dest) {
     if (len == 0) {
         return;
@@ -509,3 +511,5 @@ int mp_spiflash_cached_write(mp_spiflash_t *self, uint32_t addr, size_t len, con
     mp_spiflash_release_bus(self);
     return 0;
 }
+
+#endif // MICROPY_HW_SPIFLASH_ENABLE_CACHE

drivers/memory/spiflash.h

@@ -38,6 +38,7 @@ enum {
 
 struct _mp_spiflash_t;
 
+#if MICROPY_HW_SPIFLASH_ENABLE_CACHE
 // A cache must be provided by the user in the config struct.  The same cache
 // struct can be shared by multiple SPI flash instances.
 typedef struct _mp_spiflash_cache_t {
@@ -45,6 +46,7 @@ typedef struct _mp_spiflash_cache_t {
     struct _mp_spiflash_t *user; // current user of buf, for shared use
     uint32_t block; // current block stored in buf; 0xffffffff if invalid
 } mp_spiflash_cache_t;
+#endif
 
 typedef struct _mp_spiflash_config_t {
     uint32_t bus_kind;
@@ -59,7 +61,9 @@ typedef struct _mp_spiflash_config_t {
             const mp_qspi_proto_t *proto;
         } u_qspi;
     } bus;
+    #if MICROPY_HW_SPIFLASH_ENABLE_CACHE
     mp_spiflash_cache_t *cache; // can be NULL if cache functions not used
+    #endif
 } mp_spiflash_config_t;
 
 typedef struct _mp_spiflash_t {
@@ -75,9 +79,11 @@ int mp_spiflash_erase_block(mp_spiflash_t *self, uint32_t addr);
 void mp_spiflash_read(mp_spiflash_t *self, uint32_t addr, size_t len, uint8_t *dest);
 int mp_spiflash_write(mp_spiflash_t *self, uint32_t addr, size_t len, const uint8_t *src);
 
+#if MICROPY_HW_SPIFLASH_ENABLE_CACHE
 // These functions use the cache (which must already be configured)
 void mp_spiflash_cache_flush(mp_spiflash_t *self);
 void mp_spiflash_cached_read(mp_spiflash_t *self, uint32_t addr, size_t len, uint8_t *dest);
 int mp_spiflash_cached_write(mp_spiflash_t *self, uint32_t addr, size_t len, const uint8_t *src);
+#endif
 
 #endif // MICROPY_INCLUDED_DRIVERS_MEMORY_SPIFLASH_H

drivers/nrf24l01/nrf24l01test.py

@@ -1,6 +1,6 @@
 """Test for nrf24l01 module.  Portable between MicroPython targets."""
 
-import sys
+import usys
 import ustruct as struct
 import utime
 from machine import Pin, SPI
@@ -14,14 +14,14 @@ _RX_POLL_DELAY = const(15)
 # master may be a slow device. Value tested with Pyboard, ESP32 and ESP8266.
 _SLAVE_SEND_DELAY = const(10)
 
-if sys.platform == "pyboard":
+if usys.platform == "pyboard":
     cfg = {"spi": 2, "miso": "Y7", "mosi": "Y8", "sck": "Y6", "csn": "Y5", "ce": "Y4"}
-elif sys.platform == "esp8266":  # Hardware SPI
+elif usys.platform == "esp8266":  # Hardware SPI
     cfg = {"spi": 1, "miso": 12, "mosi": 13, "sck": 14, "csn": 4, "ce": 5}
-elif sys.platform == "esp32":  # Software SPI
+elif usys.platform == "esp32":  # Software SPI
     cfg = {"spi": -1, "miso": 32, "mosi": 33, "sck": 25, "csn": 26, "ce": 27}
 else:
-    raise ValueError("Unsupported platform {}".format(sys.platform))
+    raise ValueError("Unsupported platform {}".format(usys.platform))
 
 # Addresses are in little-endian format. They correspond to big-endian
 # 0xf0f0f0f0e1, 0xf0f0f0f0d2

examples/bluetooth/ble_bonding_peripheral.py

@@ -0,0 +1,194 @@
+# This example demonstrates a simple temperature sensor peripheral.
+#
+# The sensor's local value updates every second, and it will notify
+# any connected central every 10 seconds.
+#
+# Work-in-progress demo of implementing bonding and passkey auth.
+
+import bluetooth
+import random
+import struct
+import time
+import json
+import binascii
+from ble_advertising import advertising_payload
+
+from micropython import const
+
+_IRQ_CENTRAL_CONNECT = const(1)
+_IRQ_CENTRAL_DISCONNECT = const(2)
+_IRQ_GATTS_INDICATE_DONE = const(20)
+
+_IRQ_ENCRYPTION_UPDATE = const(28)
+_IRQ_PASSKEY_ACTION = const(31)
+
+_IRQ_GET_SECRET = const(29)
+_IRQ_SET_SECRET = const(30)
+
+_FLAG_READ = const(0x0002)
+_FLAG_NOTIFY = const(0x0010)
+_FLAG_INDICATE = const(0x0020)
+
+_FLAG_READ_ENCRYPTED = const(0x0200)
+
+# org.bluetooth.service.environmental_sensing
+_ENV_SENSE_UUID = bluetooth.UUID(0x181A)
+# org.bluetooth.characteristic.temperature
+_TEMP_CHAR = (
+    bluetooth.UUID(0x2A6E),
+    _FLAG_READ | _FLAG_NOTIFY | _FLAG_INDICATE | _FLAG_READ_ENCRYPTED,
+)
+_ENV_SENSE_SERVICE = (
+    _ENV_SENSE_UUID,
+    (_TEMP_CHAR,),
+)
+
+# org.bluetooth.characteristic.gap.appearance.xml
+_ADV_APPEARANCE_GENERIC_THERMOMETER = const(768)
+
+_IO_CAPABILITY_DISPLAY_ONLY = const(0)
+_IO_CAPABILITY_DISPLAY_YESNO = const(1)
+_IO_CAPABILITY_KEYBOARD_ONLY = const(2)
+_IO_CAPABILITY_NO_INPUT_OUTPUT = const(3)
+_IO_CAPABILITY_KEYBOARD_DISPLAY = const(4)
+
+_PASSKEY_ACTION_INPUT = const(2)
+_PASSKEY_ACTION_DISP = const(3)
+_PASSKEY_ACTION_NUMCMP = const(4)
+
+
+class BLETemperature:
+    def __init__(self, ble, name="mpy-temp"):
+        self._ble = ble
+        self._load_secrets()
+        self._ble.irq(self._irq)
+        self._ble.config(bond=True)
+        self._ble.config(le_secure=True)
+        self._ble.config(mitm=True)
+        self._ble.config(io=_IO_CAPABILITY_DISPLAY_YESNO)
+        self._ble.active(True)
+        self._ble.config(addr_mode=2)
+        ((self._handle,),) = self._ble.gatts_register_services((_ENV_SENSE_SERVICE,))
+        self._connections = set()
+        self._payload = advertising_payload(
+            name=name, services=[_ENV_SENSE_UUID], appearance=_ADV_APPEARANCE_GENERIC_THERMOMETER
+        )
+        self._advertise()
+
+    def _irq(self, event, data):
+        # Track connections so we can send notifications.
+        if event == _IRQ_CENTRAL_CONNECT:
+            conn_handle, _, _ = data
+            self._connections.add(conn_handle)
+        elif event == _IRQ_CENTRAL_DISCONNECT:
+            conn_handle, _, _ = data
+            self._connections.remove(conn_handle)
+            self._save_secrets()
+            # Start advertising again to allow a new connection.
+            self._advertise()
+        elif event == _IRQ_ENCRYPTION_UPDATE:
+            conn_handle, encrypted, authenticated, bonded, key_size = data
+            print("encryption update", conn_handle, encrypted, authenticated, bonded, key_size)
+        elif event == _IRQ_PASSKEY_ACTION:
+            conn_handle, action, passkey = data
+            print("passkey action", conn_handle, action, passkey)
+            if action == _PASSKEY_ACTION_NUMCMP:
+                accept = int(input("accept? "))
+                self._ble.gap_passkey(conn_handle, action, accept)
+            elif action == _PASSKEY_ACTION_DISP:
+                print("displaying 123456")
+                self._ble.gap_passkey(conn_handle, action, 123456)
+            elif action == _PASSKEY_ACTION_INPUT:
+                print("prompting for passkey")
+                passkey = int(input("passkey? "))
+                self._ble.gap_passkey(conn_handle, action, passkey)
+            else:
+                print("unknown action")
+        elif event == _IRQ_GATTS_INDICATE_DONE:
+            conn_handle, value_handle, status = data
+        elif event == _IRQ_SET_SECRET:
+            sec_type, key, value = data
+            key = sec_type, bytes(key)
+            value = bytes(value) if value else None
+            print("set secret:", key, value)
+            if value is None:
+                if key in self._secrets:
+                    del self._secrets[key]
+                    return True
+                else:
+                    return False
+            else:
+                self._secrets[key] = value
+            return True
+        elif event == _IRQ_GET_SECRET:
+            sec_type, index, key = data
+            print("get secret:", sec_type, index, bytes(key) if key else None)
+            if key is None:
+                i = 0
+                for (t, _key), value in self._secrets.items():
+                    if t == sec_type:
+                        if i == index:
+                            return value
+                        i += 1
+                return None
+            else:
+                key = sec_type, bytes(key)
+                return self._secrets.get(key, None)
+
+    def set_temperature(self, temp_deg_c, notify=False, indicate=False):
+        # Data is sint16 in degrees Celsius with a resolution of 0.01 degrees Celsius.
+        # Write the local value, ready for a central to read.
+        self._ble.gatts_write(self._handle, struct.pack("<h", int(temp_deg_c * 100)))
+        if notify or indicate:
+            for conn_handle in self._connections:
+                if notify:
+                    # Notify connected centrals.
+                    self._ble.gatts_notify(conn_handle, self._handle)
+                if indicate:
+                    # Indicate connected centrals.
+                    self._ble.gatts_indicate(conn_handle, self._handle)
+
+    def _advertise(self, interval_us=500000):
+        self._ble.config(addr_mode=2)
+        self._ble.gap_advertise(interval_us, adv_data=self._payload)
+
+    def _load_secrets(self):
+        self._secrets = {}
+        try:
+            with open("secrets.json", "r") as f:
+                entries = json.load(f)
+                for sec_type, key, value in entries:
+                    self._secrets[sec_type, binascii.a2b_base64(key)] = binascii.a2b_base64(value)
+        except:
+            print("no secrets available")
+
+    def _save_secrets(self):
+        try:
+            with open("secrets.json", "w") as f:
+                json_secrets = [
+                    (sec_type, binascii.b2a_base64(key), binascii.b2a_base64(value))
+                    for (sec_type, key), value in self._secrets.items()
+                ]
+                json.dump(json_secrets, f)
+        except:
+            print("failed to save secrets")
+
+
+def demo():
+    ble = bluetooth.BLE()
+    temp = BLETemperature(ble)
+
+    t = 25
+    i = 0
+
+    while True:
+        # Write every second, notify every 10 seconds.
+        i = (i + 1) % 10
+        temp.set_temperature(t, notify=i == 0, indicate=False)
+        # Random walk the temperature.
+        t += random.uniform(-0.5, 0.5)
+        time.sleep_ms(1000)
+
+
+if __name__ == "__main__":
+    demo()

examples/bluetooth/ble_simple_central.py

@@ -45,7 +45,7 @@ class BLESimpleCentral:
     def __init__(self, ble):
         self._ble = ble
         self._ble.active(True)
-        self._ble.irq(handler=self._irq)
+        self._ble.irq(self._irq)
 
         self._reset()
 

examples/bluetooth/ble_simple_peripheral.py

@@ -12,14 +12,19 @@ _IRQ_CENTRAL_CONNECT = const(1)
 _IRQ_CENTRAL_DISCONNECT = const(2)
 _IRQ_GATTS_WRITE = const(3)
 
+_FLAG_READ = const(0x0002)
+_FLAG_WRITE_NO_RESPONSE = const(0x0004)
+_FLAG_WRITE = const(0x0008)
+_FLAG_NOTIFY = const(0x0010)
+
 _UART_UUID = bluetooth.UUID("6E400001-B5A3-F393-E0A9-E50E24DCCA9E")
 _UART_TX = (
     bluetooth.UUID("6E400003-B5A3-F393-E0A9-E50E24DCCA9E"),
-    bluetooth.FLAG_READ | bluetooth.FLAG_NOTIFY,
+    _FLAG_READ | _FLAG_NOTIFY,
 )
 _UART_RX = (
     bluetooth.UUID("6E400002-B5A3-F393-E0A9-E50E24DCCA9E"),
-    bluetooth.FLAG_WRITE | bluetooth.FLAG_WRITE_NO_RESPONSE,
+    _FLAG_WRITE | _FLAG_WRITE_NO_RESPONSE,
 )
 _UART_SERVICE = (
     _UART_UUID,
@@ -31,7 +36,7 @@ class BLESimplePeripheral:
     def __init__(self, ble, name="mpy-uart"):
         self._ble = ble
         self._ble.active(True)
-        self._ble.irq(handler=self._irq)
+        self._ble.irq(self._irq)
         ((self._handle_tx, self._handle_rx),) = self._ble.gatts_register_services((_UART_SERVICE,))
         self._connections = set()
         self._write_callback = None

examples/bluetooth/ble_temperature.py

@@ -15,12 +15,16 @@ _IRQ_CENTRAL_CONNECT = const(1)
 _IRQ_CENTRAL_DISCONNECT = const(2)
 _IRQ_GATTS_INDICATE_DONE = const(20)
 
+_FLAG_READ = const(0x0002)
+_FLAG_NOTIFY = const(0x0010)
+_FLAG_INDICATE = const(0x0020)
+
 # org.bluetooth.service.environmental_sensing
 _ENV_SENSE_UUID = bluetooth.UUID(0x181A)
 # org.bluetooth.characteristic.temperature
 _TEMP_CHAR = (
     bluetooth.UUID(0x2A6E),
-    bluetooth.FLAG_READ | bluetooth.FLAG_NOTIFY | bluetooth.FLAG_INDICATE,
+    _FLAG_READ | _FLAG_NOTIFY | _FLAG_INDICATE,
 )
 _ENV_SENSE_SERVICE = (
     _ENV_SENSE_UUID,
@@ -35,7 +39,7 @@ class BLETemperature:
     def __init__(self, ble, name="mpy-temp"):
         self._ble = ble
         self._ble.active(True)
-        self._ble.irq(handler=self._irq)
+        self._ble.irq(self._irq)
         ((self._handle,),) = self._ble.gatts_register_services((_ENV_SENSE_SERVICE,))
         self._connections = set()
         self._payload = advertising_payload(

examples/bluetooth/ble_temperature_central.py

@@ -56,7 +56,7 @@ class BLETemperatureCentral:
     def __init__(self, ble):
         self._ble = ble
         self._ble.active(True)
-        self._ble.irq(handler=self._irq)
+        self._ble.irq(self._irq)
 
         self._reset()
 

examples/bluetooth/ble_uart_peripheral.py

@@ -9,14 +9,17 @@ _IRQ_CENTRAL_CONNECT = const(1)
 _IRQ_CENTRAL_DISCONNECT = const(2)
 _IRQ_GATTS_WRITE = const(3)
 
+_FLAG_WRITE = const(0x0008)
+_FLAG_NOTIFY = const(0x0010)
+
 _UART_UUID = bluetooth.UUID("6E400001-B5A3-F393-E0A9-E50E24DCCA9E")
 _UART_TX = (
     bluetooth.UUID("6E400003-B5A3-F393-E0A9-E50E24DCCA9E"),
-    bluetooth.FLAG_NOTIFY,
+    _FLAG_NOTIFY,
 )
 _UART_RX = (
     bluetooth.UUID("6E400002-B5A3-F393-E0A9-E50E24DCCA9E"),
-    bluetooth.FLAG_WRITE,
+    _FLAG_WRITE,
 )
 _UART_SERVICE = (
     _UART_UUID,
@@ -31,7 +34,7 @@ class BLEUART:
     def __init__(self, ble, name="mpy-uart", rxbuf=100):
         self._ble = ble
         self._ble.active(True)
-        self._ble.irq(handler=self._irq)
+        self._ble.irq(self._irq)
         ((self._tx_handle, self._rx_handle),) = self._ble.gatts_register_services((_UART_SERVICE,))
         # Increase the size of the rx buffer and enable append mode.
         self._ble.gatts_set_buffer(self._rx_handle, rxbuf, True)

examples/embedding/Makefile.upylib

@@ -21,6 +21,9 @@ CWARN = -Wall -Werror
 CWARN += -Wpointer-arith -Wuninitialized
 CFLAGS = $(INC) $(CWARN) -std=gnu99 -DUNIX $(CFLAGS_MOD) $(COPT) $(CFLAGS_EXTRA)
 
+# Some systems (eg MacOS) need -fno-common so that mp_state_ctx is placed in the BSS.
+CFLAGS += -fno-common
+
 # Debugging/Optimization
 ifdef DEBUG
 CFLAGS += -g
@@ -133,7 +136,6 @@ SRC_C = $(addprefix ports/unix/,\
 	gccollect.c \
 	unix_mphal.c \
 	input.c \
-	file.c \
 	modmachine.c \
 	modos.c \
 	moduselect.c \
@@ -146,6 +148,7 @@ SRC_C = $(addprefix ports/unix/,\
 LIB_SRC_C = $(addprefix lib/,\
 	$(LIB_SRC_C_EXTRA) \
 	utils/printf.c \
+	utils/gchelper_generic.c \
 	timeutils/timeutils.c \
 	)
 

examples/embedding/hello-embed.c

@@ -53,7 +53,7 @@ mp_obj_t execute_from_str(const char *str) {
 
 int main() {
     // Initialized stack limit
-    mp_stack_set_limit(40000 * (BYTES_PER_WORD / 4));
+    mp_stack_set_limit(40000 * (sizeof(void *) / 4));
     // Initialize heap
     gc_init(heap, heap + sizeof(heap));
     // Initialize interpreter

examples/rp2/pio_1hz.py

@@ -0,0 +1,35 @@
+# Example using PIO to blink an LED and raise an IRQ at 1Hz.
+
+import time
+from machine import Pin
+import rp2
+
+
+@rp2.asm_pio(set_init=rp2.PIO.OUT_LOW)
+def blink_1hz():
+    # fmt: off
+    # Cycles: 1 + 1 + 6 + 32 * (30 + 1) = 1000
+    irq(rel(0))
+    set(pins, 1)
+    set(x, 31)                  [5]
+    label("delay_high")
+    nop()                       [29]
+    jmp(x_dec, "delay_high")
+
+    # Cycles: 1 + 7 + 32 * (30 + 1) = 1000
+    set(pins, 0)
+    set(x, 31)                  [6]
+    label("delay_low")
+    nop()                       [29]
+    jmp(x_dec, "delay_low")
+    # fmt: on
+
+
+# Create the StateMachine with the blink_1hz program, outputting on Pin(25).
+sm = rp2.StateMachine(0, blink_1hz, freq=2000, set_base=Pin(25))
+
+# Set the IRQ handler to print the millisecond timestamp.
+sm.irq(lambda p: print(time.ticks_ms()))
+
+# Start the StateMachine.
+sm.active(1)

examples/rp2/pio_exec.py

@@ -0,0 +1,27 @@
+# Example using PIO to turn on an LED via an explicit exec.
+#
+# Demonstrates:
+#   - using set_init and set_base
+#   - using StateMachine.exec
+
+import time
+from machine import Pin
+import rp2
+
+# Define an empty program that uses a single set pin.
+@rp2.asm_pio(set_init=rp2.PIO.OUT_LOW)
+def prog():
+    pass
+
+
+# Construct the StateMachine, binding Pin(25) to the set pin.
+sm = rp2.StateMachine(0, prog, set_base=Pin(25))
+
+# Turn on the set pin via an exec instruction.
+sm.exec("set(pins, 1)")
+
+# Sleep for 500ms.
+time.sleep(0.5)
+
+# Turn off the set pin via an exec instruction.
+sm.exec("set(pins, 0)")

examples/rp2/pio_pinchange.py

@@ -0,0 +1,46 @@
+# Example using PIO to wait for a pin change and raise an IRQ.
+#
+# Demonstrates:
+#   - PIO wrapping
+#   - PIO wait instruction, waiting on an input pin
+#   - PIO irq instruction, in blocking mode with relative IRQ number
+#   - setting the in_base pin for a StateMachine
+#   - setting an irq handler for a StateMachine
+#   - instantiating 2x StateMachine's with the same program and different pins
+
+import time
+from machine import Pin
+import rp2
+
+
+@rp2.asm_pio()
+def wait_pin_low():
+    wrap_target()
+
+    wait(0, pin, 0)
+    irq(block, rel(0))
+    wait(1, pin, 0)
+
+    wrap()
+
+
+def handler(sm):
+    # Print a (wrapping) timestamp, and the state machine object.
+    print(time.ticks_ms(), sm)
+
+
+# Instantiate StateMachine(0) with wait_pin_low program on Pin(16).
+pin16 = Pin(16, Pin.IN, Pin.PULL_UP)
+sm0 = rp2.StateMachine(0, wait_pin_low, in_base=pin16)
+sm0.irq(handler)
+
+# Instantiate StateMachine(1) with wait_pin_low program on Pin(17).
+pin17 = Pin(17, Pin.IN, Pin.PULL_UP)
+sm1 = rp2.StateMachine(1, wait_pin_low, in_base=pin17)
+sm1.irq(handler)
+
+# Start the StateMachine's running.
+sm0.active(1)
+sm1.active(1)
+
+# Now, when Pin(16) or Pin(17) is pulled low a message will be printed to the REPL.

examples/rp2/pio_pwm.py

@@ -0,0 +1,45 @@
+# Example of using PIO for PWM, and fading the brightness of an LED
+
+from machine import Pin
+from rp2 import PIO, StateMachine, asm_pio
+from time import sleep
+
+
+@asm_pio(sideset_init=PIO.OUT_LOW)
+def pwm_prog():
+    # fmt: off
+    pull(noblock) .side(0)
+    mov(x, osr)  # Keep most recent pull data stashed in X, for recycling by noblock
+    mov(y, isr)  # ISR must be preloaded with PWM count max
+    label("pwmloop")
+    jmp(x_not_y, "skip")
+    nop()         .side(1)
+    label("skip")
+    jmp(y_dec, "pwmloop")
+    # fmt: on
+
+
+class PIOPWM:
+    def __init__(self, sm_id, pin, max_count, count_freq):
+        self._sm = StateMachine(sm_id, pwm_prog, freq=2 * count_freq, sideset_base=Pin(pin))
+        # Use exec() to load max count into ISR
+        self._sm.put(max_count)
+        self._sm.exec("pull()")
+        self._sm.exec("mov(isr, osr)")
+        self._sm.active(1)
+        self._max_count = max_count
+
+    def set(self, value):
+        # Minimum value is -1 (completely turn off), 0 actually still produces narrow pulse
+        value = max(value, -1)
+        value = min(value, self._max_count)
+        self._sm.put(value)
+
+
+# Pin 25 is LED on Pico boards
+pwm = PIOPWM(0, 25, max_count=(1 << 16) - 1, count_freq=10_000_000)
+
+while True:
+    for i in range(256):
+        pwm.set(i ** 2)
+        sleep(0.01)

examples/rp2/pio_uart_rx.py

@@ -0,0 +1,105 @@
+# Example using PIO to create a UART RX interface.
+#
+# To make it work you'll need a wire connecting GPIO4 and GPIO3.
+#
+# Demonstrates:
+#   - PIO shifting in data on a pin
+#   - PIO jmp(pin) instruction
+#   - PIO irq handler
+#   - using the second core via _thread
+
+import _thread
+from machine import Pin, UART
+from rp2 import PIO, StateMachine, asm_pio
+
+UART_BAUD = 9600
+
+HARD_UART_TX_PIN = Pin(4, Pin.OUT)
+PIO_RX_PIN = Pin(3, Pin.IN, Pin.PULL_UP)
+
+
+@asm_pio(
+    autopush=True,
+    push_thresh=8,
+    in_shiftdir=rp2.PIO.SHIFT_RIGHT,
+    fifo_join=PIO.JOIN_RX,
+)
+def uart_rx_mini():
+    # fmt: off
+    # Wait for start bit
+    wait(0, pin, 0)
+    # Preload bit counter, delay until eye of first data bit
+    set(x, 7)                 [10]
+    # Loop 8 times
+    label("bitloop")
+    # Sample data
+    in_(pins, 1)
+    # Each iteration is 8 cycles
+    jmp(x_dec, "bitloop")     [6]
+    # fmt: on
+
+
+@asm_pio(
+    in_shiftdir=rp2.PIO.SHIFT_RIGHT,
+)
+def uart_rx():
+    # fmt: off
+    label("start")
+    # Stall until start bit is asserted
+    wait(0, pin, 0)
+    # Preload bit counter, then delay until halfway through
+    # the first data bit (12 cycles incl wait, set).
+    set(x, 7)                 [10]
+    label("bitloop")
+    # Shift data bit into ISR
+    in_(pins, 1)
+    # Loop 8 times, each loop iteration is 8 cycles
+    jmp(x_dec, "bitloop")     [6]
+    # Check stop bit (should be high)
+    jmp(pin, "good_stop")
+    # Either a framing error or a break. Set a sticky flag
+    # and wait for line to return to idle state.
+    irq(block, 4)
+    wait(1, pin, 0)
+    # Don't push data if we didn't see good framing.
+    jmp("start")
+    # No delay before returning to start; a little slack is
+    # important in case the TX clock is slightly too fast.
+    label("good_stop")
+    push(block)
+    # fmt: on
+
+
+# The handler for a UART break detected by the PIO.
+def handler(sm):
+    print("break", time.ticks_ms(), end=" ")
+
+
+# Function for core1 to execute to write to the given UART.
+def core1_task(uart, text):
+    uart.write(text)
+
+
+# Set up the hard UART we're going to use to print characters.
+uart = UART(1, UART_BAUD, tx=HARD_UART_TX_PIN)
+
+for pio_prog in ("uart_rx_mini", "uart_rx"):
+    # Set up the state machine we're going to use to receive the characters.
+    sm = StateMachine(
+        0,
+        globals()[pio_prog],
+        freq=8 * UART_BAUD,
+        in_base=PIO_RX_PIN,  # For WAIT, IN
+        jmp_pin=PIO_RX_PIN,  # For JMP
+    )
+    sm.irq(handler)
+    sm.active(1)
+
+    # Tell core 1 to print some text to UART 1
+    text = "Hello, world from PIO, using {}!".format(pio_prog)
+    _thread.start_new_thread(core1_task, (uart, text))
+
+    # Echo characters received from PIO to the console.
+    for i in range(len(text)):
+        print(chr(sm.get() >> 24), end="")
+    print()

examples/rp2/pio_uart_tx.py

@@ -0,0 +1,44 @@
+# Example using PIO to create a UART TX interface
+
+from machine import Pin
+from rp2 import PIO, StateMachine, asm_pio
+
+UART_BAUD = 115200
+PIN_BASE = 10
+NUM_UARTS = 8
+
+
+@asm_pio(sideset_init=PIO.OUT_HIGH, out_init=PIO.OUT_HIGH, out_shiftdir=PIO.SHIFT_RIGHT)
+def uart_tx():
+    # fmt: off
+    # Block with TX deasserted until data available
+    pull()
+    # Initialise bit counter, assert start bit for 8 cycles
+    set(x, 7)  .side(0)       [7]
+    # Shift out 8 data bits, 8 execution cycles per bit
+    label("bitloop")
+    out(pins, 1)              [6]
+    jmp(x_dec, "bitloop")
+    # Assert stop bit for 8 cycles total (incl 1 for pull())
+    nop()      .side(1)       [6]
+    # fmt: on
+
+
+# Now we add 8 UART TXs, on pins 10 to 17. Use the same baud rate for all of them.
+uarts = []
+for i in range(NUM_UARTS):
+    sm = StateMachine(
+        i, uart_tx, freq=8 * UART_BAUD, sideset_base=Pin(PIN_BASE + i), out_base=Pin(PIN_BASE + i)
+    )
+    sm.active(1)
+    uarts.append(sm)
+
+# We can print characters from each UART by pushing them to the TX FIFO
+def pio_uart_print(sm, s):
+    for c in s:
+        sm.put(ord(c))
+
+
+# Print a different message from each UART
+for i, u in enumerate(uarts):
+    pio_uart_print(u, "Hello from UART {}!\n".format(i))

examples/rp2/pio_ws2812.py

@@ -0,0 +1,59 @@
+# Example using PIO to drive a set of WS2812 LEDs.
+
+import array, time
+from machine import Pin
+import rp2
+
+# Configure the number of WS2812 LEDs.
+NUM_LEDS = 8
+
+
+@rp2.asm_pio(
+    sideset_init=rp2.PIO.OUT_LOW,
+    out_shiftdir=rp2.PIO.SHIFT_LEFT,
+    autopull=True,
+    pull_thresh=24,
+)
+def ws2812():
+    # fmt: off
+    T1 = 2
+    T2 = 5
+    T3 = 3
+    wrap_target()
+    label("bitloop")
+    out(x, 1)               .side(0)    [T3 - 1]
+    jmp(not_x, "do_zero")   .side(1)    [T1 - 1]
+    jmp("bitloop")          .side(1)    [T2 - 1]
+    label("do_zero")
+    nop()                   .side(0)    [T2 - 1]
+    wrap()
+    # fmt: on
+
+
+# Create the StateMachine with the ws2812 program, outputting on Pin(22).
+sm = rp2.StateMachine(0, ws2812, freq=8_000_000, sideset_base=Pin(22))
+
+# Start the StateMachine, it will wait for data on its FIFO.
+sm.active(1)
+
+# Display a pattern on the LEDs via an array of LED RGB values.
+ar = array.array("I", [0 for _ in range(NUM_LEDS)])
+
+# Cycle colours.
+for i in range(4 * NUM_LEDS):
+    for j in range(NUM_LEDS):
+        r = j * 100 // (NUM_LEDS - 1)
+        b = 100 - j * 100 // (NUM_LEDS - 1)
+        if j != i % NUM_LEDS:
+            r >>= 3
+            b >>= 3
+        ar[j] = r << 16 | b
+    sm.put(ar, 8)
+    time.sleep_ms(50)
+
+# Fade out.
+for i in range(24):
+    for j in range(NUM_LEDS):
+        ar[j] = ar[j] >> 1 & 0x7F7F7F
+    sm.put(ar, 8)
+    time.sleep_ms(50)

examples/rp2/pwm_fade.py

@@ -0,0 +1,25 @@
+# Example using PWM to fade an LED.
+
+import time
+from machine import Pin, PWM
+
+
+# Construct PWM object, with LED on Pin(25).
+pwm = PWM(Pin(25))
+
+# Set the PWM frequency.
+pwm.freq(1000)
+
+# Fade the LED in and out a few times.
+duty = 0
+direction = 1
+for _ in range(8 * 256):
+    duty += direction
+    if duty > 255:
+        duty = 255
+        direction = -1
+    elif duty < 0:
+        duty = 0
+        direction = 1
+    pwm.duty_u16(duty * duty)
+    time.sleep(0.001)

examples/usercmodule/cexample/examplemodule.c

@@ -0,0 +1,37 @@
+// Include MicroPython API.
+#include "py/runtime.h"
+
+// This is the function which will be called from Python as cexample.add_ints(a, b).
+STATIC mp_obj_t example_add_ints(mp_obj_t a_obj, mp_obj_t b_obj) {
+    // Extract the ints from the micropython input objects.
+    int a = mp_obj_get_int(a_obj);
+    int b = mp_obj_get_int(b_obj);
+
+    // Calculate the addition and convert to MicroPython object.
+    return mp_obj_new_int(a + b);
+}
+// Define a Python reference to the function above.
+STATIC MP_DEFINE_CONST_FUN_OBJ_2(example_add_ints_obj, example_add_ints);
+
+// Define all properties of the module.
+// Table entries are key/value pairs of the attribute name (a string)
+// and the MicroPython object reference.
+// All identifiers and strings are written as MP_QSTR_xxx and will be
+// optimized to word-sized integers by the build system (interned strings).
+STATIC const mp_rom_map_elem_t example_module_globals_table[] = {
+    { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_cexample) },
+    { MP_ROM_QSTR(MP_QSTR_add_ints), MP_ROM_PTR(&example_add_ints_obj) },
+};
+STATIC MP_DEFINE_CONST_DICT(example_module_globals, example_module_globals_table);
+
+// Define module object.
+const mp_obj_module_t example_user_cmodule = {
+    .base = { &mp_type_module },
+    .globals = (mp_obj_dict_t *)&example_module_globals,
+};
+
+// Register the module to make it available in Python.
+// Note: the "1" in the third argument means this module is always enabled.
+// This "1" can be optionally replaced with a macro like MODULE_CEXAMPLE_ENABLED
+// which can then be used to conditionally enable this module.
+MP_REGISTER_MODULE(MP_QSTR_cexample, example_user_cmodule, 1);

examples/usercmodule/cexample/micropython.cmake

@@ -0,0 +1,15 @@
+# Create an INTERFACE library for our C module.
+add_library(usermod_cexample INTERFACE)
+
+# Add our source files to the lib
+target_sources(usermod_cexample INTERFACE
+    ${CMAKE_CURRENT_LIST_DIR}/examplemodule.c
+)
+
+# Add the current directory as an include directory.
+target_include_directories(usermod_cexample INTERFACE
+    ${CMAKE_CURRENT_LIST_DIR}
+)
+
+# Link our INTERFACE library to the usermod target.
+target_link_libraries(usermod INTERFACE usermod_cexample)

examples/usercmodule/cexample/micropython.mk

@@ -0,0 +1,9 @@
+EXAMPLE_MOD_DIR := $(USERMOD_DIR)
+
+# Add all C files to SRC_USERMOD.
+SRC_USERMOD += $(EXAMPLE_MOD_DIR)/examplemodule.c
+
+# We can add our module folder to include paths if needed
+# This is not actually needed in this example.
+CFLAGS_USERMOD += -I$(EXAMPLE_MOD_DIR)
+CEXAMPLE_MOD_DIR := $(USERMOD_DIR)

examples/usercmodule/cppexample/example.cpp

@@ -0,0 +1,17 @@
+extern "C" {
+#include <examplemodule.h>
+
+// Here we implement the function using C++ code, but since it's
+// declaration has to be compatible with C everything goes in extern "C" scope.
+mp_obj_t cppfunc(mp_obj_t a_obj, mp_obj_t b_obj) {
+    // Prove we have (at least) C++11 features.
+    const auto a = mp_obj_get_int(a_obj);
+    const auto b = mp_obj_get_int(b_obj);
+    const auto sum = [&]() {
+        return mp_obj_new_int(a + b);
+    } ();
+    // Prove we're being scanned for QSTRs.
+    mp_obj_t tup[] = {sum, MP_ROM_QSTR(MP_QSTR_hellocpp)};
+    return mp_obj_new_tuple(2, tup);
+}
+}

examples/usercmodule/cppexample/examplemodule.c

@@ -0,0 +1,28 @@
+#include <examplemodule.h>
+
+// Define a Python reference to the function we'll make available.
+// See example.cpp for the definition.
+STATIC MP_DEFINE_CONST_FUN_OBJ_2(cppfunc_obj, cppfunc);
+
+// Define all properties of the module.
+// Table entries are key/value pairs of the attribute name (a string)
+// and the MicroPython object reference.
+// All identifiers and strings are written as MP_QSTR_xxx and will be
+// optimized to word-sized integers by the build system (interned strings).
+STATIC const mp_rom_map_elem_t cppexample_module_globals_table[] = {
+    { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_cppexample) },
+    { MP_ROM_QSTR(MP_QSTR_cppfunc), MP_ROM_PTR(&cppfunc_obj) },
+};
+STATIC MP_DEFINE_CONST_DICT(cppexample_module_globals, cppexample_module_globals_table);
+
+// Define module object.
+const mp_obj_module_t cppexample_user_cmodule = {
+    .base = { &mp_type_module },
+    .globals = (mp_obj_dict_t *)&cppexample_module_globals,
+};
+
+// Register the module to make it available in Python.
+// Note: the "1" in the third argument means this module is always enabled.
+// This "1" can be optionally replaced with a macro like MODULE_CPPEXAMPLE_ENABLED
+// which can then be used to conditionally enable this module.
+MP_REGISTER_MODULE(MP_QSTR_cppexample, cppexample_user_cmodule, 1);

examples/usercmodule/cppexample/examplemodule.h

@@ -0,0 +1,5 @@
+// Include MicroPython API.
+#include "py/runtime.h"
+
+// Declare the function we'll make available in Python as cppexample.cppfunc().
+extern mp_obj_t cppfunc(mp_obj_t a_obj, mp_obj_t b_obj);

examples/usercmodule/cppexample/micropython.cmake

@@ -0,0 +1,16 @@
+# Create an INTERFACE library for our CPP module.
+add_library(usermod_cppexample INTERFACE)
+
+# Add our source files to the library.
+target_sources(usermod_cppexample INTERFACE
+    ${CMAKE_CURRENT_LIST_DIR}/example.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/examplemodule.c
+)
+
+# Add the current directory as an include directory.
+target_include_directories(usermod_cppexample INTERFACE
+    ${CMAKE_CURRENT_LIST_DIR}
+)
+
+# Link our INTERFACE library to the usermod target.
+target_link_libraries(usermod INTERFACE usermod_cppexample)

examples/usercmodule/cppexample/micropython.mk

@@ -0,0 +1,12 @@
+CPPEXAMPLE_MOD_DIR := $(USERMOD_DIR)
+
+# Add our source files to the respective variables.
+SRC_USERMOD += $(CPPEXAMPLE_MOD_DIR)/examplemodule.c
+SRC_USERMOD_CXX += $(CPPEXAMPLE_MOD_DIR)/example.cpp
+
+# Add our module directory to the include path.
+CFLAGS_USERMOD += -I$(CPPEXAMPLE_MOD_DIR)
+CXXFLAGS_USERMOD += -I$(CPPEXAMPLE_MOD_DIR)
+
+# We use C++ features so have to link against the standard library.
+LDFLAGS_USERMOD += -lstdc++

examples/usercmodule/micropython.cmake

@@ -0,0 +1,11 @@
+# This top-level micropython.cmake is responsible for listing
+# the individual modules we want to include.
+# Paths are absolute, and ${CMAKE_CURRENT_LIST_DIR} can be
+# used to prefix subdirectories.
+
+# Add the C example.
+include(${CMAKE_CURRENT_LIST_DIR}/cexample/micropython.cmake)
+
+# Add the CPP example.
+include(${CMAKE_CURRENT_LIST_DIR}/cppexample/micropython.cmake)
+

extmod/btstack/btstack.mk

@@ -11,6 +11,8 @@ EXTMOD_SRC_C += extmod/btstack/modbluetooth_btstack.c
 INC += -I$(TOP)/$(BTSTACK_EXTMOD_DIR)
 
 CFLAGS_MOD += -DMICROPY_BLUETOOTH_BTSTACK=1
+CFLAGS_MOD += -DMICROPY_PY_BLUETOOTH_USE_SYNC_EVENTS=1
+CFLAGS_MOD += -DMICROPY_PY_BLUETOOTH_ENABLE_PAIRING_BONDING=1
 
 BTSTACK_DIR = $(TOP)/lib/btstack
 
@@ -30,14 +32,33 @@ SRC_BTSTACK = \
 	$(addprefix lib/btstack/src/ble/, $(filter-out %_tlv.c, $(SRC_BLE_FILES))) \
 	lib/btstack/platform/embedded/btstack_run_loop_embedded.c
 
+ifeq ($(MICROPY_BLUETOOTH_BTSTACK_USB),1)
+ifeq ($(MICROPY_BLUETOOTH_BTSTACK_H4),1)
+	$(error Cannot specifiy both MICROPY_BLUETOOTH_BTSTACK_USB and MICROPY_BLUETOOTH_BTSTACK_H4)
+endif
+endif
+
 ifeq ($(MICROPY_BLUETOOTH_BTSTACK_USB),1)
 SRC_BTSTACK += \
 	lib/btstack/platform/libusb/hci_transport_h2_libusb.c
 
+CFLAGS_MOD += -DMICROPY_BLUETOOTH_BTSTACK_USB=1
+
 CFLAGS  += $(shell pkg-config libusb-1.0 --cflags)
 LDFLAGS += $(shell pkg-config libusb-1.0 --libs)
 endif
 
+ifeq ($(MICROPY_BLUETOOTH_BTSTACK_H4),1)
+SRC_BTSTACK += \
+	lib/btstack/src/hci_transport_h4.c \
+	lib/btstack/chipset/zephyr/btstack_chipset_zephyr.c
+
+EXTMOD_SRC_C += \
+	extmod/btstack/btstack_hci_uart.c \
+
+CFLAGS_MOD += -DMICROPY_BLUETOOTH_BTSTACK_H4=1
+endif
+
 ifeq ($(MICROPY_BLUETOOTH_BTSTACK_ENABLE_CLASSIC),1)
 include $(BTSTACK_DIR)/src/classic/Makefile.inc
 SRC_BTSTACK += \
@@ -47,11 +68,12 @@ endif
 LIB_SRC_C += $(SRC_BTSTACK)
 
 # Suppress some warnings.
-BTSTACK_WARNING_CFLAGS = -Wno-old-style-definition -Wno-unused-variable -Wno-unused-parameter
+BTSTACK_WARNING_CFLAGS = -Wno-old-style-definition -Wno-unused-variable -Wno-unused-parameter -Wimplicit-fallthrough=0
 ifneq ($(CC),clang)
 BTSTACK_WARNING_CFLAGS += -Wno-format
 endif
 $(BUILD)/lib/btstack/src/%.o: CFLAGS += $(BTSTACK_WARNING_CFLAGS)
+$(BUILD)/lib/btstack/platform/%.o: CFLAGS += $(BTSTACK_WARNING_CFLAGS)
 
 endif
 endif

extmod/btstack/btstack_config.h

@@ -8,6 +8,7 @@
 // #define ENABLE_CLASSIC
 #define ENABLE_LE_DATA_CHANNELS
 // #define ENABLE_LOG_INFO
+// #define ENABLE_LOG_DEBUG
 #define ENABLE_LOG_ERROR
 
 // BTstack configuration. buffers, sizes, ...

extmod/btstack/btstack_hci_uart.c

@@ -4,6 +4,7 @@
  * The MIT License (MIT)
  *
  * Copyright (c) 2020 Damien P. George
+ * Copyright (c) 2020 Jim Mussared
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -31,12 +32,19 @@
 #if MICROPY_PY_BLUETOOTH && MICROPY_BLUETOOTH_BTSTACK
 
 #include "lib/btstack/src/btstack.h"
-#include "lib/btstack/platform/embedded/btstack_run_loop_embedded.h"
-#include "lib/btstack/platform/embedded/hal_cpu.h"
-#include "lib/btstack/platform/embedded/hal_time_ms.h"
 
-#include "extmod/modbluetooth_hci.h"
-#include "extmod/btstack/modbluetooth_btstack.h"
+#include "extmod/mpbthci.h"
+#include "extmod/btstack/btstack_hci_uart.h"
+
+#include "mpbtstackport.h"
+
+#define HCI_TRACE (0)
+#define COL_OFF "\033[0m"
+#define COL_GREEN "\033[0;32m"
+#define COL_BLUE "\033[0;34m"
+
+// Implements a btstack btstack_uart_block_t on top of the mphciuart.h
+// interface to an HCI UART provided by the port.
 
 // We pass the bytes directly to the UART during a send, but then notify btstack in the next poll.
 STATIC bool send_done;
@@ -47,43 +55,37 @@ STATIC uint8_t *recv_buf;
 STATIC size_t recv_len;
 STATIC size_t recv_idx;
 STATIC void (*recv_handler)(void);
-
-// The IRQ functionality in btstack_run_loop_embedded.c is not used, so the
-// following three functions are empty.
-
-void hal_cpu_disable_irqs(void) {
-}
-
-void hal_cpu_enable_irqs(void) {
-}
-
-void hal_cpu_enable_irqs_and_sleep(void) {
-}
-
-uint32_t hal_time_ms(void) {
-    return mp_hal_ticks_ms();
-}
+STATIC bool init_success = false;
 
 STATIC int btstack_uart_init(const btstack_uart_config_t *uart_config) {
+    (void)uart_config;
+
     send_done = false;
     recv_len = 0;
     recv_idx = 0;
     recv_handler = NULL;
     send_handler = NULL;
 
-    // Set up the UART periperhal.
-    mp_bluetooth_hci_uart_init(MICROPY_HW_BLE_UART_ID);
+    // Set up the UART peripheral, attach IRQ and power up the HCI controller.
+    if (mp_bluetooth_hci_uart_init(MICROPY_HW_BLE_UART_ID, MICROPY_HW_BLE_UART_BAUDRATE)) {
+        init_success = false;
+        return -1;
+    }
+    if (mp_bluetooth_hci_controller_init()) {
+        init_success = false;
+        return -1;
+    }
 
+    init_success = true;
     return 0;
 }
 
 STATIC int btstack_uart_open(void) {
-    // Attach IRQ and power up the HCI controller.
-    mp_bluetooth_hci_uart_activate();
-    return 0;
+    return init_success ? 0 : 1;
 }
 
 STATIC int btstack_uart_close(void) {
+    mp_bluetooth_hci_controller_deinit();
     return 0;
 }
 
@@ -101,10 +103,12 @@ STATIC int btstack_uart_set_baudrate(uint32_t baudrate) {
 }
 
 STATIC int btstack_uart_set_parity(int parity) {
+    (void)parity;
     return 0;
 }
 
 STATIC int btstack_uart_set_flowcontrol(int flowcontrol) {
+    (void)flowcontrol;
     return 0;
 }
 
@@ -114,6 +118,14 @@ STATIC void btstack_uart_receive_block(uint8_t *buf, uint16_t len) {
 }
 
 STATIC void btstack_uart_send_block(const uint8_t *buf, uint16_t len) {
+    #if HCI_TRACE
+    printf(COL_GREEN "< [% 8d] %02x", (int)mp_hal_ticks_ms(), buf[0]);
+    for (size_t i = 1; i < len; ++i) {
+        printf(":%02x", buf[i]);
+    }
+    printf(COL_OFF "\n");
+    #endif
+
     mp_bluetooth_hci_uart_write(buf, len);
     send_done = true;
 }
@@ -123,14 +135,16 @@ STATIC int btstack_uart_get_supported_sleep_modes(void) {
 }
 
 STATIC void btstack_uart_set_sleep(btstack_uart_sleep_mode_t sleep_mode) {
+    (void)sleep_mode;
     // printf("btstack_uart_set_sleep %u\n", sleep_mode);
 }
 
 STATIC void btstack_uart_set_wakeup_handler(void (*wakeup_handler)(void)) {
+    (void)wakeup_handler;
     // printf("btstack_uart_set_wakeup_handler\n");
 }
 
-STATIC const btstack_uart_block_t btstack_uart_block = {
+const btstack_uart_block_t mp_bluetooth_btstack_hci_uart_block = {
     &btstack_uart_init,
     &btstack_uart_open,
     &btstack_uart_close,
@@ -146,15 +160,9 @@ STATIC const btstack_uart_block_t btstack_uart_block = {
     &btstack_uart_set_wakeup_handler,
 };
 
-STATIC const hci_transport_config_uart_t hci_transport_config_uart = {
-    HCI_TRANSPORT_CONFIG_UART,
-    MICROPY_HW_BLE_UART_BAUDRATE,
-    3000000,
-    0,
-    NULL,
-};
+void mp_bluetooth_btstack_hci_uart_process(void) {
+    bool host_wake = mp_bluetooth_hci_controller_woken();
 
-STATIC void btstack_uart_process(void) {
     if (send_done) {
         // If we'd done a TX in the last interval, notify btstack that it's complete.
         send_done = false;
@@ -169,6 +177,13 @@ STATIC void btstack_uart_process(void) {
     while (recv_idx < recv_len && (chr = mp_bluetooth_hci_uart_readchar()) >= 0) {
         recv_buf[recv_idx++] = chr;
         if (recv_idx == recv_len) {
+            #if HCI_TRACE
+            printf(COL_BLUE "> [% 8d] %02x", (int)mp_hal_ticks_ms(), recv_buf[0]);
+            for (size_t i = 1; i < recv_len; ++i) {
+                printf(":%02x", recv_buf[i]);
+            }
+            printf(COL_OFF "\n");
+            #endif
             recv_idx = 0;
             recv_len = 0;
             if (recv_handler) {
@@ -176,48 +191,10 @@ STATIC void btstack_uart_process(void) {
             }
         }
     }
-}
 
-void mp_bluetooth_hci_poll(void) {
-    if (mp_bluetooth_btstack_state == MP_BLUETOOTH_BTSTACK_STATE_OFF) {
-        return;
-    }
-
-    // Process uart data.
-    bool host_wake = mp_bluetooth_hci_controller_woken();
-    btstack_uart_process();
     if (host_wake) {
         mp_bluetooth_hci_controller_sleep_maybe();
     }
-
-    // Call the BTstack run loop.
-    btstack_run_loop_embedded_execute_once();
-}
-
-void mp_bluetooth_btstack_port_init(void) {
-    static bool run_loop_init = false;
-    if (!run_loop_init) {
-        run_loop_init = true;
-        btstack_run_loop_init(btstack_run_loop_embedded_get_instance());
-    } else {
-        btstack_run_loop_embedded_get_instance()->init();
-    }
-
-    // hci_dump_open(NULL, HCI_DUMP_STDOUT);
-    const hci_transport_t *transport = hci_transport_h4_instance(&btstack_uart_block);
-    hci_init(transport, &hci_transport_config_uart);
-
-    // TODO: Probably not necessary for BCM (we have our own firmware loader),
-    // but might be worth investigating for other controllers in the future.
-    // hci_set_chipset(btstack_chipset_bcm_instance());
-}
-
-void mp_bluetooth_btstack_port_deinit(void) {
-    hci_power_control(HCI_POWER_OFF);
-}
-
-void mp_bluetooth_btstack_port_start(void) {
-    hci_power_control(HCI_POWER_ON);
 }
 
 #endif // MICROPY_PY_BLUETOOTH && MICROPY_BLUETOOTH_BTSTACK

extmod/btstack/btstack_hci_uart.h

@@ -3,7 +3,8 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2019 Damien P. George
+ * Copyright (c) 2020 Damien P. George
+ * Copyright (c) 2020 Jim Mussared
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -23,18 +24,16 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#ifndef MICROPY_INCLUDED_EXTMOD_NIMBLE_NIMBLE_NIMBLE_HCI_UART_H
-#define MICROPY_INCLUDED_EXTMOD_NIMBLE_NIMBLE_NIMBLE_HCI_UART_H
 
-// Extensions to extmod/modbluetooth_hci.h specific to NimBLE.
+#ifndef MICROPY_INCLUDED_EXTMOD_BTSTACK_HCI_UART_H
+#define MICROPY_INCLUDED_EXTMOD_BTSTACK_HCI_UART_H
 
-#include "extmod/nimble/hal/hal_uart.h"
+#include "lib/btstack/src/btstack.h"
 
-// Helpers called from ports.
-void mp_bluetooth_nimble_hci_uart_process(void);
+// --- Used by the port to create the HCI transport ---------------------------
+extern const btstack_uart_block_t mp_bluetooth_btstack_hci_uart_block;
 
-// Must be provided by the port.
-void mp_bluetooth_nimble_hci_uart_rx(hal_uart_rx_cb_t rx_cb, void *rx_arg);
-void mp_bluetooth_nimble_hci_uart_tx_strn(const char *str, uint len);
+// --- Called by the MicroPython port when UART data is available -------------
+void mp_bluetooth_btstack_hci_uart_process(void);
 
-#endif // MICROPY_INCLUDED_EXTMOD_NIMBLE_NIMBLE_NIMBLE_HCI_UART_H
+#endif // MICROPY_INCLUDED_EXTMOD_BTSTACK_MODBLUETOOTH_BTSTACK_H

extmod/btstack/modbluetooth_btstack.h

@@ -56,6 +56,7 @@ enum {
     MP_BLUETOOTH_BTSTACK_STATE_OFF,
     MP_BLUETOOTH_BTSTACK_STATE_STARTING,
     MP_BLUETOOTH_BTSTACK_STATE_ACTIVE,
+    MP_BLUETOOTH_BTSTACK_STATE_HALTING,
     MP_BLUETOOTH_BTSTACK_STATE_TIMEOUT,
 };
 

extmod/extmod.cmake

@@ -0,0 +1,93 @@
+# CMake fragment for MicroPython extmod component
+
+set(MICROPY_EXTMOD_DIR "${MICROPY_DIR}/extmod")
+set(MICROPY_OOFATFS_DIR "${MICROPY_DIR}/lib/oofatfs")
+
+set(MICROPY_SOURCE_EXTMOD
+    ${MICROPY_DIR}/lib/embed/abort_.c
+    ${MICROPY_DIR}/lib/utils/printf.c
+    ${MICROPY_EXTMOD_DIR}/machine_i2c.c
+    ${MICROPY_EXTMOD_DIR}/machine_mem.c
+    ${MICROPY_EXTMOD_DIR}/machine_pulse.c
+    ${MICROPY_EXTMOD_DIR}/machine_signal.c
+    ${MICROPY_EXTMOD_DIR}/machine_spi.c
+    ${MICROPY_EXTMOD_DIR}/modbluetooth.c
+    ${MICROPY_EXTMOD_DIR}/modbtree.c
+    ${MICROPY_EXTMOD_DIR}/modframebuf.c
+    ${MICROPY_EXTMOD_DIR}/modonewire.c
+    ${MICROPY_EXTMOD_DIR}/moduasyncio.c
+    ${MICROPY_EXTMOD_DIR}/modubinascii.c
+    ${MICROPY_EXTMOD_DIR}/moducryptolib.c
+    ${MICROPY_EXTMOD_DIR}/moductypes.c
+    ${MICROPY_EXTMOD_DIR}/moduhashlib.c
+    ${MICROPY_EXTMOD_DIR}/moduheapq.c
+    ${MICROPY_EXTMOD_DIR}/modujson.c
+    ${MICROPY_EXTMOD_DIR}/modurandom.c
+    ${MICROPY_EXTMOD_DIR}/modure.c
+    ${MICROPY_EXTMOD_DIR}/moduselect.c
+    ${MICROPY_EXTMOD_DIR}/modussl_axtls.c
+    ${MICROPY_EXTMOD_DIR}/modussl_mbedtls.c
+    ${MICROPY_EXTMOD_DIR}/modutimeq.c
+    ${MICROPY_EXTMOD_DIR}/moduwebsocket.c
+    ${MICROPY_EXTMOD_DIR}/moduzlib.c
+    ${MICROPY_EXTMOD_DIR}/modwebrepl.c
+    ${MICROPY_EXTMOD_DIR}/uos_dupterm.c
+    ${MICROPY_EXTMOD_DIR}/utime_mphal.c
+    ${MICROPY_EXTMOD_DIR}/vfs.c
+    ${MICROPY_EXTMOD_DIR}/vfs_blockdev.c
+    ${MICROPY_EXTMOD_DIR}/vfs_fat.c
+    ${MICROPY_EXTMOD_DIR}/vfs_fat_diskio.c
+    ${MICROPY_EXTMOD_DIR}/vfs_fat_file.c
+    ${MICROPY_EXTMOD_DIR}/vfs_lfs.c
+    ${MICROPY_EXTMOD_DIR}/vfs_posix.c
+    ${MICROPY_EXTMOD_DIR}/vfs_posix_file.c
+    ${MICROPY_EXTMOD_DIR}/vfs_reader.c
+    ${MICROPY_EXTMOD_DIR}/virtpin.c
+    ${MICROPY_EXTMOD_DIR}/nimble/modbluetooth_nimble.c
+)
+
+# Library for btree module and associated code
+
+set(MICROPY_LIB_BERKELEY_DIR "${MICROPY_DIR}/lib/berkeley-db-1.xx")
+
+if(EXISTS "${MICROPY_LIB_BERKELEY_DIR}/btree/bt_close.c")
+    add_library(micropy_extmod_btree OBJECT
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_close.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_conv.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_debug.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_delete.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_get.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_open.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_overflow.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_page.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_put.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_search.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_seq.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_split.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_utils.c
+        ${MICROPY_LIB_BERKELEY_DIR}/mpool/mpool.c
+    )
+
+    target_include_directories(micropy_extmod_btree PRIVATE
+        ${MICROPY_LIB_BERKELEY_DIR}/PORT/include
+    )
+
+    target_compile_definitions(micropy_extmod_btree PRIVATE
+        __DBINTERFACE_PRIVATE=1
+        mpool_error=printf
+        abort=abort_
+        "virt_fd_t=void*"
+    )
+
+    # The include directories and compile definitions below are needed to build
+    # modbtree.c and should be added to the main MicroPython target.
+
+    list(APPEND MICROPY_INC_CORE
+        "${MICROPY_LIB_BERKELEY_DIR}/PORT/include"
+    )
+
+    list(APPEND MICROPY_DEF_CORE
+        __DBINTERFACE_PRIVATE=1
+        "virt_fd_t=void*"
+    )
+endif()

extmod/extmod.mk

@@ -37,6 +37,8 @@ SRC_MOD += $(addprefix $(LITTLEFS_DIR)/,\
 	lfs2.c \
 	lfs2_util.c \
 	)
+
+$(BUILD)/$(LITTLEFS_DIR)/lfs2.o: CFLAGS += -Wno-missing-field-initializers
 endif
 
 ################################################################################