Hard fault: memory access or instruction error.

[psitem]

CircuitPython version

Adafruit CircuitPython 9.0.3 on 2024-04-04; Waveshare ESP32-S3-Zero with ESP32S3
Board ID:waveshare_esp32_s3_zero
UID:437BAD95F6CC

Libraries updated to adafruit-circuitpython-bundle-9.x-mpy-20240411.zip. Plus async_button library from https://github.com/furbrain/CircuitPython_async_button/releases/download/1.2.3/circuitpython-async-button-py-1.2.3.zip

Code/REPL

import time
import board
import busio
import supervisor
from digitalio import DigitalInOut, Direction, Pull
from pwmio import PWMOut
from adafruit_motor import motor as Motor

import gc
import rtc
import displayio
import adafruit_displayio_ssd1306
import adafruit_ina219
import wifi

import adafruit_ds3231
import adafruit_ntp
import socketpool
import asyncio
from async_button import Button, MultiButton

extended_debug = False
timeset = False

displayio.release_displays()
scl_pin = board.IO2
sda_pin = board.IO1
i2c = busio.I2C(scl_pin, sda_pin)

rtc_module = adafruit_ds3231.DS3231(i2c)

ina219 = adafruit_ina219.INA219(i2c)

ina219.bus_adc_resolution = adafruit_ina219.ADCResolution.ADCRES_12BIT_16S               
ina219.shunt_adc_resolution = adafruit_ina219.ADCResolution.ADCRES_12BIT_16S               

maxcurrent = 300
maxovercurrent = 4
mincurrent = 50
maxundercurrent = 10

light_state = False

if (True):
    display_bus = displayio.I2CDisplay(i2c, device_address=0x3C)
    display = adafruit_displayio_ssd1306.SSD1306(display_bus, width=128, height=64)
    display.root_group[0].hidden = False
    display.root_group[1].hidden = True
    display.root_group[2].hidden = True
    try:
        display.root_group.remove(display.root_group[2])
        display.root_group.remove(display.root_group[1])
    except Exception as e:
        # print(e)
        e = None

    supervisor.reset_terminal(display.width, 84)
    display.root_group[0].y = 3
    display.root_group[0].x = 0


drv8833_ain1 = PWMOut(board.IO5, frequency=100)
drv8833_ain2 = PWMOut(board.IO4, frequency=100)

drv8833_enable = DigitalInOut(board.IO3)
drv8833_enable.direction = Direction.OUTPUT
drv8833_enable.value = True

# Note: Inverted, goes low for Fault conditions.
drv8833_fault = DigitalInOut(board.IO6)
drv8833_fault.direction = Direction.INPUT
drv8833_fault.pull = Pull.UP

motor_a = Motor.DCMotor(drv8833_ain1, drv8833_ain2)
motor_a.throttle = None

def print_timestamp():
    print("Time:       " + f'{time.localtime().tm_hour:02d} {time.localtime().tm_min:02d} {time.localtime().tm_sec:02d}')

CLICK_NAMES = {
    Button.SINGLE: "Single click",
    Button.DOUBLE: "Double click",
    Button.TRIPLE: "Triple click",
    Button.LONG: "Long click",
    Button.PRESSED: "Pressed",
    Button.RELEASED: "Released",
}

door_moving = False
door_direction = None
door_closed = True
door_opened = False

# 0 = stopped, positive = opening, negative = closing
door_throttle = 0 
lastDirection = 1

DOOR_STATES = {
    "Opened": 1,
    "Closed": 0,
}

async def click_watcher(button: MultiButton):
    global door_moving
    while True:
        button_name, click = await button.wait(BTN1=Button.ANY_CLICK , BTN2=Button.ANY_CLICK, BTN3=Button.ANY_CLICK)
        print(f"{button_name}: {CLICK_NAMES[click]}")

async def async_stop_btn(button: MultiButton):
    global door_throttle
    while True:
        await button.wait(BTN2=Button.SINGLE)
        print_timestamp()
        print("Stop.")
        door_throttle = 0
        await asyncio.sleep(0.10)
        print("Throttle: " + str(motor_a.throttle))

async def async_open_btn(button: MultiButton):
    global door_throttle
    while True:
        await button.wait(BTN3=Button.SINGLE)
        door_throttle = 1

async def async_close_btn(button: MultiButton):
    global door_throttle
    while True:
        await button.wait(BTN4=Button.SINGLE)
        door_throttle = -1

async def async_light_btn(button: MultiButton):
    global light_state
    while True:
        button_name, click = await button.wait(BTN1=Button.SINGLE)
        light_state = not light_state
        print("Light: " + str(light_state))
        asyncio.sleep(0.25)

async def async_kill_switch(button: MultiButton):
    global lastDirection 
    global motor_a
    global door_throttle

    while True:
        button_name, click = await button.wait(BTN5=Button.PRESSED)
        print_timestamp()
        print("Kill switch.")
        door_throttle = None
        asyncio.sleep(0.100)
        print("Opening door.")
        door_throttle = -1

async def async_open_close_btn(button: MultiButton):
    global lastDirection 
    global motor_a
    global door_throttle

    while True:
        button_name, click = await button.wait(BTN3=Button.SINGLE)
        print("Last: " + str(lastDirection))
        if ( (door_throttle is None) or (door_throttle == 0) ):
            print("Starting")
            door_throttle = -lastDirection
            lastDirection = door_throttle
        else:
            lastDirection = door_throttle
            door_throttle = 0

            print("Starting.")
            print("Throttle: " + str(motor_a.throttle))

        print("Now: " + str(lastDirection))

async def async_move_door(throttle: int):
    global motor_a
    global door_throttle

    motor_a.throttle = throttle

    while True:
        if (door_throttle == 0):
            motor_a.throttle = None
        else:
            motor_a.throttle = door_throttle
        await asyncio.sleep(0.05)


#async def async_monitor_current(duration: int):
async def async_monitor_current():
    global motor_a
    global ina219
    global door_throttle
    global lastDirection
    # print_timestamp()
    overcount = 0
    undercount = 0
    while (True):
        # now = time.monotonic()  # Time in seconds since power on
        # while (now + duration) > time.monotonic():
        while ( ( door_throttle != 0 ) ):
            while ( (overcount < maxovercurrent) and (undercount < maxundercurrent) ):
                await asyncio.sleep(0.005)
                current = ina219.current  # current in mA
                # print("Current: {:5.0f}  mA".format(current))

                if ( current > maxcurrent ):
                    overcount += 1
                    print("Current: {:5.0f}  mA".format(current))
                    print("Over-Current Count: {:1d}".format(overcount))
                elif ( ( door_throttle != 0 ) and ( current <mincurrent )):
                    pass
                    undercount += 1
                    print("Current: {:5.0f}  mA".format(current))
                    print("Under-Current Count: {:1d}".format(undercount))
                elif ( (current < maxcurrent) and (current > mincurrent) ):
                    undercount = 0
                    overcount = 0
            print("Stopping.")
            door_throttle = 0
            overcount = 0
            undercount = 0
            print_timestamp()
            await asyncio.sleep(0.10)
            print("Throttle: " + str(motor_a.throttle))
        await asyncio.sleep(0.05)

async def async_dump_throttle():
    global motor_a
    while (True):
        # print("Throttle: " + str(motor_a.throttle))
        # print("Enable:   " + str(drv8833_enable.value))
        # print("Fault:    " + str(not drv8833_fault.value))
        # print("Memory free: " + f'{gc.mem_free():>7}')
        await asyncio.sleep(1)

async def async_monitor_fault():
    global drv8833_fault
    
    while (True):
        # Pin is inverted
        if not drv8833_fault:
            print_timestamp()
            print("Fault:    " + str(not drv8833_fault.value))
            print("Stopping due to Fault.")
            door_throttle = 0

        await asyncio.sleep(0.005)


async def async_ntp():
    global ntp
    global pool
    global timeset
    global rtc_module
    while(False):
        loopcount = 0
        while ( (not timeset) and (loopcount < 5) ):
            if ( extended_debug ):
                print("Entering NTP loop.")
                print("loopcount: " + str(loopcount))
            try:
                loopcount += 1
                rtc.RTC().datetime = ntp.datetime
                rtc_module.datetime = ntp.datetime
                timeset = True
            except Exception as e:
                pass
        await asyncio.sleep(4 * 60 * 60) # sleep 4 hours
        
async def main():
    global door_throttle
    
    # Yellow / Light
    BTN1 = Button(
        board.IO10,
        value_when_pressed=False,
        long_click_enable=True,
        long_click_min_duration=0.7,
        triple_click_enable=False,
        double_click_enable=False,
    )

    # Red / Stop
    BTN2 = Button(
        board.IO9,
        value_when_pressed=False,
        long_click_enable=True,
        long_click_min_duration=0.7,
        triple_click_enable=False,
        double_click_enable=False,
    )

    # Green / Go
    BTN3 = Button(
        board.IO8,
        value_when_pressed=False,
        long_click_enable=True,
        long_click_min_duration=0.7,
        triple_click_enable=False,
        double_click_enable=False,
    )

    # Door Kill Switch
    BTN5 = Button(
        board.IO11,
        value_when_pressed=False,
        triple_click_enable=False,
        double_click_enable=False,
    )

    multibutton = MultiButton(BTN1=BTN1, BTN2=BTN2, BTN3=BTN3, BTN5=BTN5)

    await asyncio.gather(async_open_close_btn(multibutton), 
                         async_kill_switch(multibutton), 
                         async_stop_btn(multibutton), 
                         async_light_btn(multibutton), 
                         async_monitor_current(), 
                         async_move_door(0), 
                         async_dump_throttle(), 
                         async_ntp(), 
                         async_monitor_fault(),
                         )

    print("End of Main")

rtc.RTC().datetime = rtc_module.datetime
rtc.set_time_source(rtc_module)

print(str(wifi.radio.ipv4_address))
pool = socketpool.SocketPool(wifi.radio)
ntp = adafruit_ntp.NTP(pool, tz_offset=0)
# print("Localtime: " + str(time.localtime()))
# print("RTC time:  " + str(rtc.RTC().datetime))
# print("i2c time:  " + str(rtc_module.datetime))
print("Date:       " + f'{time.localtime().tm_year:04d} {time.localtime().tm_mon:02d} {time.localtime().tm_mday:02d}')
print("Time:       " + f'{time.localtime().tm_hour:02d} {time.localtime().tm_min:02d} {time.localtime().tm_sec:02d}')

asyncio.run(main())

Behavior

Eventually the board reboots into safe mode.

Auto-reload is off.
Running in safe mode! Not running saved code.

You are in safe mode because:
CircuitPython core code crashed hard. Whoops!
Hard fault: memory access or instruction error.
Please file an issue with your program at github.com/adafruit/circuitpython/issues.
Press reset to exit safe mode.

Press any key to enter the REPL. Use CTRL-D to reload.

Description

No response

Additional information

I hooked up a USB-TTL adapter to the TX pin on the board and got this. Is there anything that can be done to output more debug info?

ESP-ROM:esp32s3-20210327
Build:Mar 27 2021
rst:0x1 (POWERON),boot:0xb (SPI_FAST_FLASH_BOOT)
SPIWP:0xee
mode:DIO, clock div:1
load:0x3fce3818,len:0x138
load:0x403c9700,len:0x4
load:0x403c9704,len:0xa9c
load:0x403cc700,len:0x26d4
entry 0x403c9870
ESP-ROM:esp32s3-20210327
Build:Mar 27 2021
rst:0x3 (RTC_SW_SYS_RST),boot:0xb (SPI_FAST_FLASH_BOOT)
Saved PC:0x4037cd5e
SPIWP:0xee
mode:DIO, clock div:1
load:0x3fce3818,len:0x138
load:0x403c9700,len:0x4
load:0x403c9704,len:0xa9c
load:0x403cc700,len:0x26d4
entry 0x403c9870

[dhalbert]

Two things:

1. Could you retry with 9.0.4 and with 9.1.0-beta.0 (beta.1 will be soon as well)?
2. Could you try removing pieces of the program to get the minimum reproducible example? For instance, try removing the PWM code, the display code, etc. Thanks.

[dhalbert]

@psitem I wired this up with a QT Py ESP32-S3 4/2 and all the same I2C devices that you have here. I am not pushing any buttons. You said "Eventually the board reboots into safe mode.". How long is eventually, and do I need to provoke it by pushing various assigned buttons?

[psitem]

After the first time it happened, it was about 2 days before the next occurrence. Subsequent occurrences were sooner and some were with no interaction with the buttons.

I've had this set aside for a bit but I've now updated to 9.0.4 and the 20240503 library bundle and will leave it running.

[psitem]

I had a reset overnight, but it didn't come back up in safe mode. Unfortunately I didn't have my USB-TTL adapter hooked up so I have no capture of what the bootloader did, but the USB came up as this:

PORT    VID     PID     DESCRIPTION
----------------------------------------------------------------------
COM33   239A    011B    USB Serial Device (COM33) [TinyUF2 CDC]

Instead of this:

PORT    VID     PID     DESCRIPTION
----------------------------------------------------------------------
COM34   303A    81B4    Espressif PID:81B4 [CircuitPython CDC control]

I'm not sure what that's about — flash mode? — but I have the USB-TTL hooked up now for the next time.

[psitem]

Yep, with 9.0.4 it's going into flash mode after the reset. Guess I'll get another issue opened and try 9.1.0-beta.1

ESP-ROM:esp32s3-20210327
Build:Mar 27 2021
rst:0x3 (RTC_SW_SYS_RST),boot:0xb (SPI_FAST_FLASH_BOOT)
Saved PC:0x4037cd5e
SPIWP:0xee
mode:DIO, clock div:1
load:0x3fce3818,len:0x1734
load:0x403c9700,len:0x4
load:0x403c9704,len:0x1184
load:0x403cc700,len:0x3090
entry 0x403c9a80
I (31) boot: ESP-IDF v5.1.1 2nd stage bootloader
I (31) boot: compile time Mar 22 2024 16:38:50
I (31) boot: Multicore bootloader
I (34) boot: chip revision: v0.1
I (38) qio_mode: Enabling default flash chip QIO
I (44) boot.esp32s3: Boot SPI Speed : 80MHz
I (48) boot.esp32s3: SPI Mode       : QIO
I (53) boot.esp32s3: SPI Flash Size : 4MB
I (58) boot: Enabling RNG early entropy source...
I (63) boot: Partition Table:
I (67) boot: ## Label            Usage          Type ST Offset   Length
I (74) boot:  0 nvs              WiFi data        01 02 00009000 00005000
I (82) boot:  1 otadata          OTA data         01 00 0000e000 00002000
I (89) boot:  2 ota_0            OTA app          00 10 00010000 00160000
I (97) boot:  3 ota_1            OTA app          00 11 00170000 00160000
I (104) boot:  4 uf2              factory app      00 00 002d0000 00040000
I (112) boot:  5 ffat             Unknown data     01 81 00310000 000f0000
I (119) boot: End of partition table
I (123) boot: Reset Reason = 3
I (127) boot: Reset hint = 4
I (131) boot: Detect double reset using RC on GPIO 34 to enter UF2 bootloader
I (138) esp_image: segment 0: paddr=002d0020 vaddr=3c020020 size=09e08h ( 40456) map
I (153) esp_image: segment 1: paddr=002d9e30 vaddr=3fc90700 size=0222ch (  8748) load
I (157) esp_image: segment 2: paddr=002dc064 vaddr=40374000 size=03fb4h ( 16308) load
I (167) esp_image: segment 3: paddr=002e0020 vaddr=42000020 size=1740ch ( 95244) map
I (187) esp_image: segment 4: paddr=002f7434 vaddr=40377fb4 size=08650h ( 34384) load
I (199) boot: Loaded app from partition at offset 0x2d0000
I (199) boot: Disabling RNG early entropy source...
I (211) cpu_start: Multicore app
I (211) cpu_start: Pro cpu up.
I (211) cpu_start: Starting app cpu, entry point is 0x40374d1c
I (0) cpu_start: App cpu up.
I (229) cpu_start: Pro cpu start user code
I (229) cpu_start: cpu freq: 160000000 Hz
I (229) cpu_start: Application information:
I (232) cpu_start: Project name:     tinyuf2
I (237) cpu_start: App version:      0.18.2
I (242) cpu_start: Compile time:     Mar 22 2024 16:38:43
I (248) cpu_start: ELF file SHA256:  db9a8c83aae3e3c0...
I (254) cpu_start: ESP-IDF:          v5.1.1
I (259) cpu_start: Min chip rev:     v0.0
I (263) cpu_start: Max chip rev:     v0.99
I (268) cpu_start: Chip rev:         v0.1
I (273) heap_init: Initializing. RAM available for dynamic allocation:
I (280) heap_init: At 3FCA6360 len 000433B0 (268 KiB): DRAM
I (286) heap_init: At 3FCE9710 len 00005724 (21 KiB): STACK/DRAM
I (293) heap_init: At 3FCF0000 len 00008000 (32 KiB): DRAM
I (299) heap_init: At 600FE010 len 00001FD8 (7 KiB): RTCRAM
I (306) spi_flash: detected chip: generic
I (310) spi_flash: flash io: qio
I (314) sleep: Configure to isolate all GPIO pins in sleep state
I (321) sleep: Enable automatic switching of GPIO sleep configuration
I (328) app_start: Starting scheduler on CPU0
2mI (3 3)33p _atnrt: StaSting scoeduler [n CPU1
I (343) main_task: Calling app_main()
I (343) gpio: GPIO[21]| InputEn: 0| OutputEn: 0| OpenDrain: 0| Pullup: 1| Pulldown: 0| Intr:0
I (353) gpio: GPIO[33]| InputEn: 0| OutputEn: 1| OpenDrain: 0| Pullup: 1| Pulldown: 0| Intr:0
I (363) main_task: Returned from app_main()

[dhalbert]

As I mentioned above I was unable to reproduce this. I have no motors connected. When this happens quiesecently, are the motors doing anything?

We would be really interested in a minimal version of this that provokes the behavior. If you can turn off various aspects of the program to see what happens, that would be helpful. Do I need to wait possibly hours for it to happen?

[psitem]

Do I need to wait possibly hours for it to happen?

It was two full days between this last crash and getting it going again from the previous crash. It had sometimes happened in less time on 9.0.2 but around two days has been consistent. Nothing was happening with the motor.

I've made two more of these setups, minus the actual motor and switch on GPIO11, that I can try to run simultaneously with code trimmed out. Are there particular areas you think are the most likely suspects? The network stuff? i2c bus?

[dhalbert]

Are there particular areas you think are the most likely suspects? The network stuff? i2c bus?

I think the network stuff and async are the most likely. The I2C and motor control are pretty unlikely, as is the minimal display code. So you code take those out to start and leave the network and async and see what happens. That is, remove the actual motor control but leave all the motor timing code in. And when it's quiescent, not much is happening except NTP every 4 hours. You could speed that up to once a minute, say, and see whether it's failing on NTP.

[psitem]

NTP isn't actually happening with the code, probably I'd disabled it to validate that the DS3231 was providing time. Re-familiarizing myself with the code, once it has started the only things happening are:

• Wi-Fi.
• The Web API is enabled but not being used.
• Some functions await-ing on async_button.button.wait() that wouldn't be doing anything when crashes occur.
• Some functions await-ing on asyncio.sleep() that wouldn't be doing anything when crashes occur.
• DRV8833 fault checks every 5ms on GPIO6 in async_monitor_fault.
• INA219 current checks in async_monitor_current, which I'd intended to only happen while the motor is operating but that's not quite how the code is written and it'll keep checking every 5ms under some conditions if the throttle was ever set. For your test setup without a motor, it should have exited both inner loops and stopped checking. For my setup, mashing buttons, it probably would not stop checking.

So I'm going to leave one hammering the INA219 and set up a second that won't and see if one is still running in 3 days.

[psitem]

Well, the original crashed again already. And come up again in flash mode, this time cycling the motor, which has not happened before and now has me concerned for leaving that running unattended. I may have inadvertently changed bootloaders when I was upgrading it to 9.1.0-beta.1 because it came up with the FTHRS3BOOT drive instead of just the TinyUF2 CDC serial port.

TinyUF2 Bootloader 0.18.2 - tinyusb (0.15.0-566-gf1e006d09)
Model: Adafruit Feather ESP32-S3
Board-ID: ESP32-S3-Feather-revC
Date: Mar 22 2024
Flash Size: 0x00160000 bytes

ESP-ROM:esp32s3-20210327
Build:Mar 27 2021
rst:0xc (RTC_SW_CPU_RST),boot:0xb (SPI_FAST_FLASH_BOOT)
Saved PC:0x400356db
SPIWP:0xee
mode:DIO, clock div:1
load:0x3fce3818,len:0x1734
load:0x403c9700,len:0x4
load:0x403c9704,len:0x1184
load:0x403cc700,len:0x3090
entry 0x403c9a80
I (31) boot: ESP-IDF v5.1.1 2nd stage bootloader
I (31) boot: compile time Mar 22 2024 16:38:50
I (32) boot: Multicore bootloader
I (35) boot: chip revision: v0.1
I (38) qio_mode: Enabling default flash chip QIO
I (44) boot.esp32s3: Boot SPI Speed : 80MHz
I (48) boot.esp32s3: SPI Mode       : QIO
I (53) boot.esp32s3: SPI Flash Size : 4MB
I (58) boot: Enabling RNG early entropy source...
I (63) boot: Partition Table:
I (67) boot: ## Label            Usage          Type ST Offset   Length
I (74) boot:  0 nvs              WiFi data        01 02 00009000 00005000
I (82) boot:  1 otadata          OTA data         01 00 0000e000 00002000
I (89) boot:  2 ota_0            OTA app          00 10 00010000 00160000
I (97) boot:  3 ota_1            OTA app          00 11 00170000 00160000
I (104) boot:  4 uf2              factory app      00 00 002d0000 00040000
I (112) boot:  5 ffat             Unknown data     01 81 00310000 000f0000
I (119) boot: End of partition table
I (123) boot: Reset Reason = 12
I (127) boot: Reset hint = 4
I (131) boot: Detect double reset using RC on GPIO 34 to enter UF2 bootloader
I (139) esp_image: segment 0: paddr=002d0020 vaddr=3c020020 size=09e08h ( 40456) map
I (153) esp_image: segment 1: paddr=002d9e30 vaddr=3fc90700 size=0222ch (  8748) load
I (157) esp_image: segment 2: paddr=002dc064 vaddr=40374000 size=03fb4h ( 16308) load
I (167) esp_image: segment 3: paddr=002e0020 vaddr=42000020 size=1740ch ( 95244) map
I (187) esp_image: segment 4: paddr=002f7434 vaddr=40377fb4 size=08650h ( 34384) load
I (199) boot: Loaded app from partition at offset 0x2d0000
I (199) boot: Disabling RNG early entropy source...
I (211) cpu_start: Multicore app
I (211) cpu_start: Pro cpu up.
I (211) cpu_start: Starting app cpu, entry point is 0x40374d1c
I (190) cpu_start: App cpu up.
I (229) cpu_start: Pro cpu start user code
I (229) cpu_start: cpu freq: 160000000 Hz
I (229) cpu_start: Application information:
I (232) cpu_start: Project name:     tinyuf2
I (237) cpu_start: App version:      0.18.2
I (242) cpu_start: Compile time:     Mar 22 2024 16:38:43
I (248) cpu_start: ELF file SHA256:  db9a8c83aae3e3c0...
I (254) cpu_start: ESP-IDF:          v5.1.1
I (259) cpu_start: Min chip rev:     v0.0
I (264) cpu_start: Max chip rev:     v0.99
I (269) cpu_start: Chip rev:         v0.1
I (273) heap_init: Initializing. RAM available for dynamic allocation:
I (281) heap_init: At 3FCA6360 len 000433B0 (268 KiB): DRAM
I (287) heap_init: At 3FCE9710 len 00005724 (21 KiB): STACK/DRAM
I (294) heap_init: At 3FCF0000 len 00008000 (32 KiB): DRAM
I (300) heap_init: At 600FE010 len 00001FD8 (7 KiB): RTCRAM
I (306) spi_flash: detected chip: generic
I (311) spi_flash: flash io: qio
I (315) sleep: Configure to isolate all GPIO pins in sleep state
I (321) sleep: Enable automatic switching of GPIO sleep configuration
I (329) app_start: Starting scheduler on CPU0
0m3 m334)34p _atnrt: StaSting scoeduler [n CPU1
I (344) main_task: Calling app_main()
I (344) gpio: GPIO[21]| InputEn: 0| OutputEn: 0| OpenDrain: 0| Pullup: 1| Pulldown: 0| Intr:0
I (354) gpio: GPIO[33]| InputEn: 0| OutputEn: 1| OpenDrain: 0| Pullup: 1| Pulldown: 0| Intr:0
I (364) main_task: Returned from app_main()