diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8dae3c6..f68b07c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,7 +4,6 @@ on: [pull_request] env: APP_NAME: 'Camera Streamer' - IDF_TARGET: 'esp32' IDF_VERSION: 'v5.5.1' IDF_COMPONENT_MANAGER: "1" # whether to enable the component manager or not FLASH_TOTAL_OVERRIDE: '2097152' # 2MB flash app partition for main app @@ -14,25 +13,34 @@ jobs: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - board: 'timer-cam' + target: 'esp32' + - board: 'xiao-esp32s3-sense' + target: 'esp32s3' + steps: - name: Checkout repo uses: actions/checkout@v4 with: submodules: true - - name: Build Main + - name: Build Main (${{ matrix.board }}) uses: espressif/esp-idf-ci-action@v1 with: esp_idf_version: ${{ env.IDF_VERSION }} - target: ${{ env.IDF_TARGET }} + target: ${{ matrix.target }} path: '.' - - name: Determine Size Delta + - name: Determine Size Delta (${{ matrix.board }}) uses: esp-cpp/esp-idf-size-delta@v1 with: - app_name: ${{ env.APP_NAME }} + app_name: '${{ env.APP_NAME }} (${{ matrix.board }})' app_path: '.' - idf_target: ${{ env.IDF_TARGET }} + idf_target: ${{ matrix.target }} idf_version: ${{ env.IDF_VERSION }} idf_component_manager: ${{ env.IDF_COMPONENT_MANAGER }} flash_total_override: ${{ env.FLASH_TOTAL_OVERRIDE }} diff --git a/.github/workflows/package_main.yml b/.github/workflows/package_main.yml index ce6dd13..282efa2 100644 --- a/.github/workflows/package_main.yml +++ b/.github/workflows/package_main.yml @@ -9,7 +9,6 @@ on: env: APP_NAME: 'Camera Streamer' - IDF_TARGET: 'esp32' IDF_VERSION: 'v5.5.1' IDF_COMPONENT_MANAGER: "1" # whether to enable the component manager or not FLASH_TOTAL_OVERRIDE: '2097152' # 2MB flash app partition for main app @@ -20,8 +19,14 @@ jobs: runs-on: ubuntu-latest continue-on-error: false - outputs: - zipfile-id: ${{ steps.zip_step.outputs.artifact-id }} + strategy: + fail-fast: false + matrix: + include: + - board: 'timer-cam' + target: 'esp32' + - board: 'xiao-esp32s3-sense' + target: 'esp32s3' steps: - name: Checkout repo @@ -29,18 +34,17 @@ jobs: with: submodules: true - - name: Build Main Code + - name: Build Main Code (${{ matrix.board }}) uses: espressif/esp-idf-ci-action@v1 with: - esp_idf_version: release-v5.4 - target: esp32 + esp_idf_version: ${{ env.IDF_VERSION }} + target: ${{ matrix.target }} path: '.' - - name: Upload Build Outputs + - name: Upload Build Outputs (${{ matrix.board }}) uses: actions/upload-artifact@v4 - id: zip_step with: - name: build-artifacts + name: build-artifacts-${{ matrix.board }} path: | build/*.bin build/*.elf @@ -49,40 +53,47 @@ jobs: build/flasher_args.json build/flash_args - - name: Attach files to release + - name: Stage release files (${{ matrix.board }}) + if: ${{ github.event.release && github.event.action == 'published' }} + run: | + mkdir -p release + cp build/camera-streamer.bin release/camera-streamer_${{ matrix.board }}.bin + cp build/camera-streamer.elf release/camera-streamer_${{ matrix.board }}.elf + cp build/bootloader/bootloader.bin release/bootloader_${{ matrix.board }}.bin + cp build/partition_table/partition-table.bin release/partition-table_${{ matrix.board }}.bin + cp build/flasher_args.json release/flasher_args_${{ matrix.board }}.json + + - name: Attach files to release (${{ matrix.board }}) uses: softprops/action-gh-release@v2 if: ${{ github.event.release && github.event.action == 'published' }} with: files: | - build/*.bin - build/*.elf - build/bootloader/bootloader.bin - build/partition_table/partition-table.bin - build/flasher_args.json - build/flash_args + release/* - - name: Determine Size Delta + - name: Determine Size Delta (${{ matrix.board }}) # only run this if the release is published if: ${{ github.event.release && github.event.action == 'published' }} uses: esp-cpp/esp-idf-size-delta@v1 with: - app_name: ${{ env.APP_NAME }} + app_name: '${{ env.APP_NAME }} (${{ matrix.board }})' app_path: "." - idf_target: ${{ env.IDF_TARGET }} + idf_target: ${{ matrix.target }} idf_version: ${{ env.IDF_VERSION }} idf_component_manager: ${{ env.IDF_COMPONENT_MANAGER }} flash_total_override: ${{ env.FLASH_TOTAL_OVERRIDE }} post_comment: 'false' package: - name: Package the binaries into an executables for Windows, MacOS, and Linux (Ubuntu) + name: Package the binaries into executables for Windows, MacOS, and Linux (Ubuntu) needs: build strategy: + fail-fast: false matrix: os: [windows-latest, macos-latest, ubuntu-latest] + board: ['timer-cam', 'xiao-esp32s3-sense'] runs-on: ${{ matrix.os }} steps: - uses: esp-cpp/esp-packaged-programmer-action@v1.0.5 with: - zipfile-id: ${{ needs.build.outputs.zipfile-id }} - programmer-name: 'camera-streamer_programmer' + zipfile-name: build-artifacts-${{ matrix.board }} + programmer-name: 'camera-streamer_${{ matrix.board }}_programmer' diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml index 4270278..f391da4 100644 --- a/.github/workflows/static_analysis.yml +++ b/.github/workflows/static_analysis.yml @@ -17,4 +17,4 @@ jobs: use_cmake: false # (Optional) cppcheck args - cppcheck_args: --force --enable=all --inline-suppr --inconclusive --platform=mips32 --std=c++17 --suppressions-list=$GITHUB_WORKSPACE/suppressions.txt + cppcheck_args: --force --enable=all --check-level=exhaustive --inline-suppr --inconclusive --platform=mips32 --std=c++17 --suppressions-list=$GITHUB_WORKSPACE/suppressions.txt diff --git a/.gitignore b/.gitignore index b4ef190..0468cd7 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ sdkconfig.old .DS_Store dependencies.lock managed_components/ +.cache/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 8f7360b..7259b02 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,12 +10,10 @@ set(EXTRA_COMPONENT_DIRS add_compile_definitions(BOARD_HAS_PSRAM) -set( - COMPONENTS - "main esptool_py esp_psram esp32-camera esp32-timer-cam mdns monitor nvs rtsp socket task wifi " - CACHE STRING - "List of components to include" - ) +# The set of board-specific components (esp32-timer-cam vs xiao-esp32s3-sense) +# depends on the target chip, so component selection is handled by the +# component manager (main/idf_component.yml) and main/CMakeLists.txt REQUIRES +# rather than a hard-coded COMPONENTS list here. project(camera-streamer) diff --git a/README.md b/README.md index d4e351a..ad806a1 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,18 @@ # camera-streamer -Example for [ESP32 TimerCam](https://github.com/m5stack/TimerCam-idf) rebuilt using [ESPP](http://github.com/esp-cpp/espp) to stream video over the network +Camera streamer built using [ESPP](http://github.com/esp-cpp/espp) to stream +video (and, where available, audio) over the network. It supports two boards, +selected at build time by the target chip: -It uses RTSP + RTP (over UDP) to perform real-time streaming of the camera data over the network to multiple clients. +| Board | Target | Sensor | Streams | +|-------|--------|--------|---------| +| [M5Stack ESP32 TimerCam](https://github.com/m5stack/TimerCam-idf) | `esp32` | OV3660 | MJPEG video | +| [Seeed Studio XIAO ESP32S3 Sense](https://wiki.seeedstudio.com/xiao_esp32s3_getting_started/) | `esp32s3` | OV2640 + PDM mic | MJPEG video + L16 PCM audio | + +It uses RTSP + RTP (over UDP) to perform real-time streaming of the camera data +over the network to multiple clients. On the XIAO ESP32S3 Sense, the onboard PDM +microphone is streamed as a second RTP track (L16 / 16 kHz mono) alongside the +video. https://user-images.githubusercontent.com/213467/236601550-ba1a5ba1-4f1c-4dfa-9b64-94afbd46ef3f.mp4 @@ -115,7 +125,25 @@ documentation](https://docs.espressif.com/projects/esp-idf/en/v5.5.1/esp32s3/get ### Build and Flash -Build the project and flash it to the board, then run monitor tool to view serial output: +This project supports two boards, each tied to a specific chip. Select the board +by setting the matching target before building — ESP-IDF automatically applies +the corresponding `sdkconfig.defaults.` (which selects the board in +Kconfig): + +``` +# M5Stack ESP32 TimerCam (the default target) +idf.py set-target esp32 + +# Seeed Studio XIAO ESP32S3 Sense (camera + microphone) +idf.py set-target esp32s3 +``` + +You can also flip the board explicitly under `Camera Streamer Configuration -> +Target board` in `idf.py menuconfig` (only the board matching the current chip is +selectable). + +Then build the project and flash it to the board, and run the monitor tool to +view serial output: ``` idf.py -p PORT flash monitor @@ -125,13 +153,24 @@ idf.py -p PORT flash monitor (To exit the serial monitor, type ``Ctrl-]``.) +> [!NOTE] +> When you switch targets, run `idf.py fullclean` first (or delete the `build/` +> and `sdkconfig` files) so the new target's defaults are applied cleanly. + See the Getting Started Guide for full steps to configure and use ESP-IDF to build projects. ## Hardware -This sample is designed to run on the ESP32 TimerCam ([Amazon Link](https://www.amazon.com/dp/B09W2RSPGL?psc=1&ref=ppx_yo2ov_dt_b_product_details)). +This sample runs on either the M5Stack ESP32 TimerCam or the Seeed Studio XIAO +ESP32S3 Sense. The board pin mappings come from the corresponding ESPP board +support package ([`esp32-timer-cam`](https://github.com/esp-cpp/espp/tree/main/components/esp32-timer-cam) +or [`xiao-esp32s3-sense`](https://github.com/esp-cpp/espp/tree/main/components/xiao-esp32s3-sense)), +so the pin tables below are informational. -The ESP32 TimerCam has the following specs: +### M5Stack ESP32 TimerCam + +The ESP32 TimerCam ([Amazon Link](https://www.amazon.com/dp/B09W2RSPGL?psc=1&ref=ppx_yo2ov_dt_b_product_details)) +has the following specs: * 8MB PSRAM * 4MB flash @@ -193,6 +232,31 @@ From their actual code... | LED | IO2 | | Button | IO37 | +### Seeed Studio XIAO ESP32S3 Sense + +The XIAO ESP32S3 Sense ([Seeed Studio Wiki](https://wiki.seeedstudio.com/xiao_esp32s3_getting_started/)) +pairs the XIAO ESP32S3 module with the "Sense" expansion board, which adds an +OV2640 camera and a PDM microphone. + +The XIAO ESP32S3 Sense has the following specs: + +* 8MB PSRAM, 8MB flash +* OV2640 image sensor (2MP), via the Sense expansion board (XCLK 20 MHz) +* PDM microphone, via the Sense expansion board +* microSD slot (SPI), via the Sense expansion board +* User LED + native USB-C (console runs over USB Serial/JTAG) + +The camera, microphone, microSD, and LED pin mappings are provided by the ESPP +[`xiao-esp32s3-sense`](https://github.com/esp-cpp/espp/tree/main/components/xiao-esp32s3-sense) +board support package. + +#### Microphone (PDM): + +| Mic Pin | ESP32-S3 GPIO Number | +|---------|----------------------| +| CLK | IO42 | +| DATA | IO41 | + ## Additional References @@ -201,3 +265,5 @@ From their actual code... * https://github.com/esp-cpp/espp * https://github.com/m5stack/TimerCam-idf * https://docs.m5stack.com/#/en/unit/timercam +* https://wiki.seeedstudio.com/xiao_esp32s3_getting_started/ +* https://esp-cpp.github.io/espp/xiao_esp32s3_sense.html diff --git a/dependencies.lock b/dependencies.lock index f815b7c..3e7a9d7 100644 --- a/dependencies.lock +++ b/dependencies.lock @@ -1,22 +1,4 @@ dependencies: - espp/adc: - component_hash: 8df5e759a2ce090b1d084c4ee5c4e9ac9a60d48a1736f524c29696bd6e99ee4d - dependencies: - - name: espp/base_component - registry_url: https://components.espressif.com - require: private - version: '>=1.0' - - name: espp/task - registry_url: https://components.espressif.com - require: private - version: '>=1.0' - - name: idf - require: private - version: '>=5.0' - source: - registry_url: https://components.espressif.com - type: service - version: 1.1.1 espp/base_component: component_hash: 753dd0037b7dcccb859480dda36eaf7ca83a3f30c466da7c60b957ac10356c8e dependencies: @@ -31,38 +13,6 @@ dependencies: registry_url: https://components.espressif.com type: service version: 1.1.1 - espp/base_peripheral: - component_hash: a6fc75fd03e90382a54cb87e45e720032fed440468111975fe596eb549f05eb9 - dependencies: - - name: espp/base_component - registry_url: https://components.espressif.com - require: private - version: '>=1.0' - - name: idf - require: private - version: '>=5.0' - source: - registry_url: https://components.espressif.com - type: service - version: 1.1.1 - espp/bm8563: - component_hash: a89718cd21cff1819ab66606526b500325cb1dd2eb0a947709df549ed0b4f3f3 - dependencies: - - name: espp/base_peripheral - registry_url: https://components.espressif.com - require: private - version: '>=1.0' - - name: espp/utils - registry_url: https://components.espressif.com - require: private - version: '>=1.0' - - name: idf - require: private - version: '>=5.0' - source: - registry_url: https://components.espressif.com - type: service - version: 1.1.1 espp/cli: component_hash: 129d81da026d387a5bade014e44184549c9d1af553f9042d74aca68b8a60f79a dependencies: @@ -73,53 +23,9 @@ dependencies: - name: idf require: private version: '>=5.0' - source: - registry_url: https://components.espressif.com - type: service - version: 1.1.1 - espp/esp32-timer-cam: - component_hash: 7ccb42130c2f4ce70ea2fbfa26a153859ae96b1eae571dbb10da562c894bee74 - dependencies: - - name: espp/adc - registry_url: https://components.espressif.com - require: private - version: '>=1.0' - - name: espp/base_component - registry_url: https://components.espressif.com - require: private - version: '>=1.0' - - name: espp/bm8563 - registry_url: https://components.espressif.com - require: private - version: '>=1.0' - - name: espp/i2c - registry_url: https://components.espressif.com - require: private - version: '>=1.0' - - name: espp/interrupt - registry_url: https://components.espressif.com - require: private - version: '>=1.0' - - name: espp/led - registry_url: https://components.espressif.com - require: private - version: '>=1.0' - - name: espp/math - registry_url: https://components.espressif.com - require: private - version: '>=1.0' - - name: espp/task - registry_url: https://components.espressif.com - require: private - version: '>=1.0' - - name: idf - require: private - version: '>=5.0' source: registry_url: https://components.espressif.com/ type: service - targets: - - esp32 version: 1.1.1 espp/format: component_hash: 306438454fb4391109c50ddb43695b8405bbd339c6e219b83694d29e67e41a04 @@ -131,46 +37,6 @@ dependencies: registry_url: https://components.espressif.com type: service version: 1.1.1 - espp/i2c: - component_hash: 38b4cbd740ad89e7d96e61f7420f48b7bd7b69a11ebfc4b66ec1ecabbeb0412b - dependencies: - - name: espp/base_component - registry_url: https://components.espressif.com - require: private - version: '>=1.0' - - name: espp/cli - registry_url: https://components.espressif.com - require: private - version: '>=1.0' - - name: espp/task - registry_url: https://components.espressif.com - require: private - version: '>=1.0' - - name: idf - require: private - version: '>=5.0' - source: - registry_url: https://components.espressif.com - type: service - version: 1.1.1 - espp/interrupt: - component_hash: d104e57d02c8f9e94709354a388d564ab13d2b958bf81f473ea4e4bf5e0fdc60 - dependencies: - - name: espp/base_component - registry_url: https://components.espressif.com - require: private - version: '>=1.0' - - name: espp/task - registry_url: https://components.espressif.com - require: private - version: '>=1.0' - - name: idf - require: private - version: '>=5.0' - source: - registry_url: https://components.espressif.com - type: service - version: 1.1.1 espp/led: component_hash: 4132466e1180e5d53efab93e1369133ec2ae250d874a45f163e9789566db1e5b dependencies: @@ -303,28 +169,44 @@ dependencies: registry_url: https://components.espressif.com/ type: service version: 1.1.1 - espp/utils: - component_hash: c392a49fe04498463dcdc9e4d36e91003b68384b334bce486b028138be7a5552 + espp/wifi: + component_hash: 3099372d99327cca94b2857731345956fa5796dfd113bd5d14bbae2ef98e71fc dependencies: + - name: espp/base_component + registry_url: https://components.espressif.com + require: private + version: '>=1.0' + - name: espp/cli + registry_url: https://components.espressif.com + require: private + version: '>=1.0' + - name: espp/nvs + registry_url: https://components.espressif.com + require: private + version: '>=1.0' - name: idf require: private version: '>=5.0' source: - registry_url: https://components.espressif.com + registry_url: https://components.espressif.com/ type: service version: 1.1.1 - espp/wifi: - component_hash: 3099372d99327cca94b2857731345956fa5796dfd113bd5d14bbae2ef98e71fc + espp/xiao-esp32s3-sense: + component_hash: b12b42e41acca46c7440cf97bd0a55a64ea774cf86ffed4b78e1c8969ae30978 dependencies: - name: espp/base_component registry_url: https://components.espressif.com require: private version: '>=1.0' - - name: espp/cli + - name: espp/led registry_url: https://components.espressif.com require: private version: '>=1.0' - - name: espp/nvs + - name: espp/math + registry_url: https://components.espressif.com + require: private + version: '>=1.0' + - name: espp/task registry_url: https://components.espressif.com require: private version: '>=1.0' @@ -334,6 +216,8 @@ dependencies: source: registry_url: https://components.espressif.com/ type: service + targets: + - esp32s3 version: 1.1.1 espressif/esp32-camera: component_hash: bc9c8a6b51df777a014fa295825b3de5069bc0300c317acff20c97cf4a10ac7d @@ -374,16 +258,17 @@ dependencies: type: idf version: 6.0.0 direct_dependencies: -- espp/esp32-timer-cam +- espp/cli - espp/monitor - espp/nvs - espp/rtsp - espp/socket - espp/task - espp/wifi +- espp/xiao-esp32s3-sense - espressif/esp32-camera - espressif/mdns - idf -manifest_hash: 54d502f6fbd9916123194044b95f82f75ac69f2fc5feae88612fd6c16af8add7 -target: esp32 +manifest_hash: 0dac9d33bea65db5a0860771b690a63eb2c86c2ac954ab6434eb150eff4e683a +target: esp32s3 version: 3.0.0 diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt index a941e22..f7b0ad9 100644 --- a/main/CMakeLists.txt +++ b/main/CMakeLists.txt @@ -1,2 +1,13 @@ +idf_build_get_property(target IDF_TARGET) + +if(target STREQUAL "esp32s3") + set(board_requires xiao-esp32s3-sense esp_driver_i2s) +else() + set(board_requires esp32-timer-cam) +endif() + idf_component_register(SRC_DIRS "." - INCLUDE_DIRS ".") + INCLUDE_DIRS "." + REQUIRES esp_psram esp32-camera mdns esp_wifi nvs_flash + cli monitor nvs rtsp socket task wifi + ${board_requires}) diff --git a/main/Kconfig.projbuild b/main/Kconfig.projbuild index a60526a..e76e956 100644 --- a/main/Kconfig.projbuild +++ b/main/Kconfig.projbuild @@ -1,11 +1,59 @@ menu "Camera Streamer Configuration" + choice CAMERA_STREAMER_BOARD + prompt "Target board" + default CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE if IDF_TARGET_ESP32S3 + default CAMERA_STREAMER_BOARD_ESP32_TIMER_CAM + help + Select which camera board this firmware targets. Each board is tied + to a specific chip, so set the matching target first + (e.g. `idf.py set-target esp32` for the TimerCam or + `idf.py set-target esp32s3` for the XIAO ESP32S3 Sense). The + sdkconfig.defaults. file selects the right board by default. + + config CAMERA_STREAMER_BOARD_ESP32_TIMER_CAM + bool "M5Stack ESP32 TimerCam (OV3660)" + depends on IDF_TARGET_ESP32 + help + M5Stack ESP32 TimerCam. Streams MJPEG video over RTSP. + + config CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE + bool "Seeed Studio XIAO ESP32S3 Sense (OV2640 + PDM mic)" + depends on IDF_TARGET_ESP32S3 + help + Seeed Studio XIAO ESP32S3 Sense. Streams MJPEG video plus L16 + PCM audio (from the onboard PDM microphone) over RTSP. + endchoice + config RTSP_SERVER_PORT int "RTSP Server Port" default 8554 help The port number of the RTSP server. + config RTSP_ACCEPT_TASK_STACK_SIZE + int "RTSP accept task stack size (bytes)" + range 2048 32768 + default 4096 + help + Stack size for the RTSP server accept task. + + config RTSP_SESSION_TASK_STACK_SIZE + int "RTSP session task stack size (bytes)" + range 2048 32768 + default 4096 + help + Stack size for the RTSP server session-dispatch task. + + config RTSP_CONTROL_TASK_STACK_SIZE + int "RTSP control task stack size (bytes)" + range 2048 32768 + default 8192 + help + Stack size for each per-client RTSP control task. Increase this if + the board reports a stack overflow in a task named like + "RtspSession ". + config ESP_WIFI_SSID string "WiFi SSID" default "" diff --git a/main/idf_component.yml b/main/idf_component.yml index f7867c1..828f211 100644 --- a/main/idf_component.yml +++ b/main/idf_component.yml @@ -11,4 +11,13 @@ dependencies: espp/socket: '>=1.0' espp/nvs: '>=1.0' espp/task: '>=1.0' - espp/esp32-timer-cam: '>=1.0' + espp/cli: '>=1.0' + ## Board support packages - only one is pulled in, based on the target chip + espp/esp32-timer-cam: + version: '>=1.0' + rules: + - if: 'target in [esp32]' + espp/xiao-esp32s3-sense: + version: '>=1.0' + rules: + - if: 'target in [esp32s3]' diff --git a/main/main.cpp b/main/main.cpp index 9347b51..5b814d0 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -1,38 +1,89 @@ #include "sdkconfig.h" +#include +#include +#include #include +#include +#include +#include #include #include #include +#include +#include +#include #include #include "cli.hpp" -#include "esp32-timer-cam.hpp" #include "heap_monitor.hpp" +#include "mjpeg_packetizer.hpp" #include "nvs.hpp" +#include "rtsp_server.hpp" #include "task.hpp" #include "task_monitor.hpp" -#include "tcp_socket.hpp" -#include "udp_socket.hpp" #include "wifi_sta.hpp" #include "wifi_sta_menu.hpp" #include "esp_camera.h" -#include "rtsp_server.hpp" +#if CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE +#include +#include + +#include "generic_packetizer.hpp" +#include "xiao-esp32s3-sense.hpp" +#else +#include "esp32-timer-cam.hpp" +#endif using namespace std::chrono_literals; +#if CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE +using Board = espp::XiaoEsp32S3Sense; +static constexpr const char *board_name = "Seeed Studio XIAO ESP32S3 Sense"; +static constexpr const char *mdns_hostname_prefix = "xiao-camera-streamer"; +static constexpr const char *mdns_instance_name = "XIAO Camera Streamer"; +#else +using Board = espp::EspTimerCam; +static constexpr const char *board_name = "M5Stack ESP32 TimerCam"; +static constexpr const char *mdns_hostname_prefix = "camera-streamer"; +static constexpr const char *mdns_instance_name = "Camera Streamer"; +#endif + static espp::Logger logger({.tag = "Camera Streamer", .level = espp::Logger::Verbosity::INFO}); namespace { constexpr auto idle_capture_poll_period = 250ms; constexpr auto dma_pressure_backoff = 250ms; -constexpr auto target_stream_period = 100ms; +constexpr auto camera_capture_error_backoff = 100ms; +constexpr auto target_video_period = 100ms; constexpr size_t min_dma_free_bytes_for_streaming = 12 * 1024; constexpr size_t min_dma_largest_block_for_streaming = 4 * 1024; + +constexpr int video_track_id = 0; +constexpr char rtsp_path[] = "mjpeg/1"; + +#if CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE +constexpr auto audio_capture_timeout = 50ms; +constexpr int audio_track_id = 1; +constexpr uint32_t microphone_sample_rate_hz = + espp::XiaoEsp32S3Sense::microphone_default_sample_rate_hz(); +constexpr size_t microphone_frame_samples = microphone_sample_rate_hz / 50; // 20 ms +constexpr size_t microphone_frame_bytes = microphone_frame_samples * sizeof(int16_t); +#endif + +// Wait until `deadline`, returning true if the task was asked to stop while waiting. +template +bool wait_until_or_stop(std::mutex &m, std::condition_variable &cv, bool &task_notified, + const std::chrono::time_point &deadline) { + std::unique_lock lk(m); + auto stop_requested = cv.wait_until(lk, deadline, [&task_notified] { return task_notified; }); + task_notified = false; + return stop_requested; +} } // namespace std::recursive_mutex server_mutex; @@ -40,17 +91,31 @@ std::unique_ptr camera_task; std::unique_ptr memory_monitor_task; std::unique_ptr task_monitor; std::shared_ptr rtsp_server; -std::atomic frames_streamed{0}; -std::atomic camera_initialized{false}; +bool mdns_started{false}; +std::atomic video_frames_streamed{0}; + +#if CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE +std::unique_ptr audio_task; +i2s_chan_handle_t microphone_rx_channel{nullptr}; +std::atomic audio_frames_streamed{0}; +#endif esp_err_t initialize_camera(void); bool start_rtsp_server(std::string_view server_address, int server_port); -bool camera_task_fn(std::mutex &m, std::condition_variable &cv); +void stop_streaming(void); +bool camera_task_fn(std::mutex &m, std::condition_variable &cv, bool &task_notified); bool memory_monitor_task_fn(std::mutex &m, std::condition_variable &cv, bool &task_notified); +#if CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE +esp_err_t initialize_microphone(void); +void deinitialize_microphone(void); +bool initialize_and_test_sdcard(void); +bool audio_task_fn(std::mutex &m, std::condition_variable &cv, bool &task_notified); +#endif + extern "C" void app_main(void) { - esp_err_t err; logger.info("Bootup"); + logger.info("Board: {}", board_name); #if CONFIG_ESP32_WIFI_NVS_ENABLED std::error_code ec; @@ -59,32 +124,50 @@ extern "C" void app_main(void) { nvs.init(ec); #endif - auto &timer_cam = espp::EspTimerCam::get(); + static constexpr float disconnected_led_breathing_period = 1.0f; + static constexpr float connected_led_breathing_period = 3.5f; + + auto &board = Board::get(); - // initialize RTC - if (!timer_cam.initialize_rtc()) { +#if !CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE + // initialize RTC (TimerCam has an onboard BM8563) + if (!board.initialize_rtc()) { logger.error("Could not initialize RTC"); return; } +#endif // initialize LED - static constexpr float disconnected_led_breathing_period = 1.0f; - static constexpr float connected_led_breathing_period = 3.5f; - if (!timer_cam.initialize_led(disconnected_led_breathing_period)) { + if (!board.initialize_led(disconnected_led_breathing_period)) { logger.error("Could not initialize LED"); return; } - timer_cam.start_led_breathing(); + board.start_led_breathing(); + +#if CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE + // initialize the (optional) microSD card; continues on failure + initialize_and_test_sdcard(); +#endif // initialize camera logger.info("Initializing camera"); - err = initialize_camera(); + auto err = initialize_camera(); if (err != ESP_OK) { logger.error("Could not initialize camera: {} '{}'", err, esp_err_to_name(err)); - } else { - camera_initialized = true; + return; } +#if CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE + // initialize microphone (XIAO ESP32S3 Sense has an onboard PDM microphone) + logger.info("Initializing microphone"); + err = initialize_microphone(); + if (err != ESP_OK) { + logger.error("Could not initialize microphone: {} '{}'", err, esp_err_to_name(err)); + deinitialize_microphone(); + return; + } +#endif + logger.info("Starting memory monitors"); task_monitor = std::make_unique(espp::TaskMonitor::Config{.period = 30s}); memory_monitor_task = espp::Task::make_unique(espp::Task::Config{ @@ -105,88 +188,93 @@ extern "C" void app_main(void) { .password = CONFIG_ESP_WIFI_PASSWORD, .num_connect_retries = CONFIG_ESP_MAXIMUM_RETRY, .on_connected = - []() { - static auto &timer_cam = espp::EspTimerCam::get(); - timer_cam.set_led_breathing_period(connected_led_breathing_period); - }, + []() { Board::get().set_led_breathing_period(connected_led_breathing_period); }, .on_disconnected = []() { - static auto &timer_cam = espp::EspTimerCam::get(); - timer_cam.set_led_breathing_period(disconnected_led_breathing_period); - // Move ownership of the task / server out from under the lock so we - // can join the camera task without holding server_mutex. The camera - // task acquires server_mutex every iteration, so joining it while - // holding the lock would deadlock. - std::unique_ptr dead_task; - std::shared_ptr dead_server; - { - std::lock_guard lock(server_mutex); - logger.info("Stopping camera task"); - dead_task = std::move(camera_task); - logger.info("Stopping RTSP server"); - dead_server = std::move(rtsp_server); - logger.info("Deiniting MDNS"); - mdns_free(); - } - // Reset (joins the task / destroys the server) with no lock held. - dead_task.reset(); - dead_server.reset(); + Board::get().set_led_breathing_period(disconnected_led_breathing_period); + logger.info("WiFi disconnected, stopping RTSP streaming"); + stop_streaming(); }, .on_got_ip = [](ip_event_got_ip_t *eventdata) { auto server_address = fmt::format("{}.{}.{}.{}", IP2STR(&eventdata->ip_info.ip)); logger.info("got IP: {}", server_address); - // create the camera and rtsp server, and the cv/m - // they'll use to communicate - std::lock_guard lock(server_mutex); - // Guard against a got-IP event arriving without an intervening - // disconnect (some reconnect paths): don't tear down a running - // server / task out from under active sessions. - if (rtsp_server) { - logger.info("RTSP server already running, ignoring duplicate got-IP event"); - return; + // start from a clean slate in case a previous session is still up + stop_streaming(); + + if (esp_wifi_set_ps(WIFI_PS_NONE) != ESP_OK) { + logger.warn( + "Could not disable WiFi power save; RTSP streaming may hit TX backpressure"); + } else { + logger.info("Disabled WiFi power save for RTSP streaming"); } + if (!start_rtsp_server(server_address, CONFIG_RTSP_SERVER_PORT)) { - logger.error("RTSP server failed to start, not starting camera task"); + logger.error("RTSP server failed to start, not starting capture tasks"); return; } - if (!camera_initialized) { - logger.error("Camera not initialized; RTSP server is up but no frames " - "will be captured"); + + logger.info("Creating camera task"); + auto new_camera_task = espp::Task::make_unique(espp::Task::Config{ + .callback = camera_task_fn, + .task_config = {.name = "Camera Task", .priority = 10}, + .log_level = espp::Logger::Verbosity::WARN, + }); + if (!new_camera_task->start()) { + logger.error("Could not start camera task"); + stop_streaming(); return; } - // initialize the camera - logger.info("Creating camera task"); - camera_task = espp::Task::make_unique( - espp::Task::Config{.callback = camera_task_fn, - .task_config = {.name = "Camera Task", .priority = 10}}); - camera_task->start(); - }}); - if (esp_wifi_set_ps(WIFI_PS_NONE) != ESP_OK) { - logger.warn("Could not disable WiFi power save; RTSP streaming may hit TX backpressure"); - } else { - logger.info("Disabled WiFi power save for RTSP streaming"); - } +#if CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE + logger.info("Creating audio task"); + auto new_audio_task = espp::Task::make_unique(espp::Task::Config{ + .callback = audio_task_fn, + .task_config = + { + .name = "Audio Task", + .stack_size_bytes = 6 * 1024, + .priority = 9, + }, + .log_level = espp::Logger::Verbosity::WARN, + }); + if (!new_audio_task->start()) { + logger.error("Could not start audio task"); + new_camera_task.reset(); + stop_streaming(); + return; + } +#endif + + std::lock_guard lock(server_mutex); + camera_task = std::move(new_camera_task); +#if CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE + audio_task = std::move(new_audio_task); +#endif + }}); espp::WifiStaMenu sta_menu(wifi_sta); auto root_menu = sta_menu.get(); root_menu->Insert( "memory", [](std::ostream &out) { - out << "Frames streamed: " << frames_streamed.load() << std::endl; + out << "Video frames streamed: " << video_frames_streamed.load() << '\n'; +#if CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE + out << "Audio frames streamed: " << audio_frames_streamed.load() << '\n'; +#endif out << espp::HeapMonitor::get_table( {MALLOC_CAP_DEFAULT, MALLOC_CAP_INTERNAL, MALLOC_CAP_SPIRAM, MALLOC_CAP_DMA}) << std::endl; }, "Display current heap monitor information."); +#if !CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE root_menu->Insert( "battery", [](std::ostream &out) { - static auto &timer_cam = espp::EspTimerCam::get(); - out << fmt::format("Battery voltage: {:.2f}\n", timer_cam.get_battery_voltage()); + out << fmt::format("Battery voltage: {:.2f}\n", Board::get().get_battery_voltage()); }, "Display the current battery voltage."); +#endif cli::Cli cli(std::move(root_menu)); cli::SetColor(); @@ -204,151 +292,333 @@ esp_err_t initialize_camera(void) { * * WQVGA: 400x240 * * HVGA: 480x320 * * VGA: 640x480 - * * WVGA: 768x480 - * * FWVGA: 854x480 * * SVGA: 800x600 - * * DVGA: 960x640 - * * WSVGA: 1024x600 * * XGA: 1024x768 - * * WXGA: 1280x800 - * * WSXGA: 1440x900 - * * SXGA: 1280x1024 * * UXGA: 1600x1200 */ + auto &board = Board::get(); - auto &timer_cam = espp::EspTimerCam::get(); +#if CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE + auto camera = board.camera_pins(); static camera_config_t camera_config = { - .pin_pwdn = -1, - .pin_reset = timer_cam.get_camera_reset_pin(), - .pin_xclk = timer_cam.get_camera_xclk_pin(), - .pin_sccb_sda = timer_cam.get_camera_sda_pin(), - .pin_sccb_scl = timer_cam.get_camera_scl_pin(), - - .pin_d7 = timer_cam.get_camera_d7_pin(), - .pin_d6 = timer_cam.get_camera_d6_pin(), - .pin_d5 = timer_cam.get_camera_d5_pin(), - .pin_d4 = timer_cam.get_camera_d4_pin(), - .pin_d3 = timer_cam.get_camera_d3_pin(), - .pin_d2 = timer_cam.get_camera_d2_pin(), - .pin_d1 = timer_cam.get_camera_d1_pin(), - .pin_d0 = timer_cam.get_camera_d0_pin(), - .pin_vsync = timer_cam.get_camera_vsync_pin(), - .pin_href = timer_cam.get_camera_href_pin(), - .pin_pclk = timer_cam.get_camera_pclk_pin(), - - // EXPERIMENTAL: Set to 16MHz on ESP32-S2 or ESP32-S3 to enable EDMA mode - .xclk_freq_hz = timer_cam.get_camera_xclk_freq_hz(), + .pin_pwdn = static_cast(camera.pwdn), + .pin_reset = static_cast(camera.reset), + .pin_xclk = static_cast(camera.xclk), + .pin_sccb_sda = static_cast(camera.sccb_sda), + .pin_sccb_scl = static_cast(camera.sccb_scl), + + .pin_d7 = static_cast(camera.d7), + .pin_d6 = static_cast(camera.d6), + .pin_d5 = static_cast(camera.d5), + .pin_d4 = static_cast(camera.d4), + .pin_d3 = static_cast(camera.d3), + .pin_d2 = static_cast(camera.d2), + .pin_d1 = static_cast(camera.d1), + .pin_d0 = static_cast(camera.d0), + .pin_vsync = static_cast(camera.vsync), + .pin_href = static_cast(camera.href), + .pin_pclk = static_cast(camera.pclk), + + .xclk_freq_hz = board.camera_xclk_freq_hz(), .ledc_timer = LEDC_TIMER_0, .ledc_channel = LEDC_CHANNEL_0, - .pixel_format = PIXFORMAT_JPEG, // YUV422,GRAYSCALE,RGB565,JPEG - .frame_size = FRAMESIZE_QVGA, // QVGA-UXGA, For ESP32, do not use sizes above QVGA when not - // JPEG. The performance of the ESP32-S series has improved a - // lot, but JPEG mode always gives better frame rates. + .pixel_format = PIXFORMAT_JPEG, + .frame_size = FRAMESIZE_QVGA, + .jpeg_quality = 15, + .fb_count = 2, + .fb_location = CAMERA_FB_IN_PSRAM, + .grab_mode = CAMERA_GRAB_LATEST, + .sccb_i2c_port = I2C_NUM_0, + }; +#else + static camera_config_t camera_config = { + .pin_pwdn = -1, + .pin_reset = board.get_camera_reset_pin(), + .pin_xclk = board.get_camera_xclk_pin(), + .pin_sccb_sda = board.get_camera_sda_pin(), + .pin_sccb_scl = board.get_camera_scl_pin(), + + .pin_d7 = board.get_camera_d7_pin(), + .pin_d6 = board.get_camera_d6_pin(), + .pin_d5 = board.get_camera_d5_pin(), + .pin_d4 = board.get_camera_d4_pin(), + .pin_d3 = board.get_camera_d3_pin(), + .pin_d2 = board.get_camera_d2_pin(), + .pin_d1 = board.get_camera_d1_pin(), + .pin_d0 = board.get_camera_d0_pin(), + .pin_vsync = board.get_camera_vsync_pin(), + .pin_href = board.get_camera_href_pin(), + .pin_pclk = board.get_camera_pclk_pin(), + + .xclk_freq_hz = board.get_camera_xclk_freq_hz(), + .ledc_timer = LEDC_TIMER_0, + .ledc_channel = LEDC_CHANNEL_0, + .pixel_format = PIXFORMAT_JPEG, + .frame_size = FRAMESIZE_QVGA, .jpeg_quality = 15, // 0-63, for OV series camera sensors, lower number means higher quality - .fb_count = 2, // When jpeg mode is used, if fb_count more than one, the driver will work in - // continuous mode. + .fb_count = 2, // continuous mode when >1 in JPEG mode .fb_location = CAMERA_FB_IN_PSRAM, - .grab_mode = - CAMERA_GRAB_LATEST, // CAMERA_GRAB_WHEN_EMPTY // . Sets when buffers should be filled + .grab_mode = CAMERA_GRAB_LATEST, .sccb_i2c_port = I2C_NUM_0}; +#endif + auto err = esp_camera_init(&camera_config); if (err != ESP_OK) { logger.error("Could not initialize camera: {} '{}'", err, esp_err_to_name(err)); return err; } - // set the mirror and flip - specific to the ESP32-TimerCam! - logger.info("Enabling camera vflip"); + + // set the mirror and flip - specific to each board's sensor orientation sensor_t *s = esp_camera_sensor_get(); if (!s) { logger.error("Could not get camera sensor handle"); return ESP_FAIL; } +#if CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE + s->set_vflip(s, false); + s->set_hmirror(s, false); +#else s->set_vflip(s, true); s->set_hmirror(s, false); +#endif + return ESP_OK; +} + +#if CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE +esp_err_t initialize_microphone(void) { + if (microphone_rx_channel != nullptr) { + return ESP_OK; + } + + auto &sense = espp::XiaoEsp32S3Sense::get(); + i2s_chan_config_t channel_config = I2S_CHANNEL_DEFAULT_CONFIG(I2S_NUM_AUTO, I2S_ROLE_MASTER); + channel_config.auto_clear = true; + channel_config.dma_desc_num = 4; + channel_config.dma_frame_num = microphone_frame_samples; + + auto err = i2s_new_channel(&channel_config, nullptr, µphone_rx_channel); + if (err != ESP_OK) { + logger.error("Could not allocate I2S RX channel: {} '{}'", err, esp_err_to_name(err)); + microphone_rx_channel = nullptr; + return err; + } + + auto microphone_config = sense.microphone_config(microphone_sample_rate_hz); + err = i2s_channel_init_pdm_rx_mode(microphone_rx_channel, µphone_config); + if (err != ESP_OK) { + logger.error("Could not initialize PDM RX mode: {} '{}'", err, esp_err_to_name(err)); + deinitialize_microphone(); + return err; + } + + err = i2s_channel_enable(microphone_rx_channel); + if (err != ESP_OK) { + logger.error("Could not enable microphone channel: {} '{}'", err, esp_err_to_name(err)); + deinitialize_microphone(); + return err; + } + + logger.info("Microphone initialized for {} Hz mono PCM", microphone_sample_rate_hz); return ESP_OK; } +void deinitialize_microphone(void) { + if (microphone_rx_channel == nullptr) { + return; + } + i2s_channel_disable(microphone_rx_channel); + i2s_del_channel(microphone_rx_channel); + microphone_rx_channel = nullptr; +} + +bool initialize_and_test_sdcard(void) { + auto &sense = espp::XiaoEsp32S3Sense::get(); + if (!sense.initialize_sdcard()) { + logger.warn("Could not initialize microSD card, continuing without storage"); + return false; + } + + constexpr char test_file_path[] = "/sdcard/xiao_sdcard_smoke_test.txt"; + constexpr std::string_view test_payload = "xiao-esp32s3-sense sdcard smoke test\n"; + + auto *file = std::fopen(test_file_path, "wb"); + if (file == nullptr) { + logger.error("Could not open '{}' for writing", test_file_path); + return false; + } + + size_t bytes_written = std::fwrite(test_payload.data(), 1, test_payload.size(), file); + std::fclose(file); + if (bytes_written != test_payload.size()) { + logger.error("microSD smoke test short write: wrote {} of {} bytes", bytes_written, + test_payload.size()); + std::remove(test_file_path); + return false; + } + + std::array readback{}; + file = std::fopen(test_file_path, "rb"); + if (file == nullptr) { + logger.error("Could not reopen '{}' for reading", test_file_path); + std::remove(test_file_path); + return false; + } + + size_t bytes_read = std::fread(readback.data(), 1, readback.size() - 1, file); + std::fclose(file); + std::remove(test_file_path); + + std::string_view readback_view(readback.data(), bytes_read); + if (readback_view != test_payload) { + logger.error("microSD smoke test readback mismatch: expected '{}', got '{}'", test_payload, + readback_view); + return false; + } + + logger.info("microSD smoke test passed using '{}'", test_file_path); + return true; +} +#endif // CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE + bool start_rtsp_server(std::string_view server_address, int server_port) { logger.info("Creating RTSP server at {}:{}", server_address, server_port); - std::lock_guard lock(server_mutex); - rtsp_server = std::make_shared( - espp::RtspServer::Config{.server_address = std::string(server_address), - .port = server_port, - .path = "mjpeg/1", - .log_level = espp::Logger::Verbosity::WARN}); - rtsp_server->set_session_log_level(espp::Logger::Verbosity::WARN); - if (!rtsp_server->start()) { + + auto server = std::make_shared(espp::RtspServer::Config{ + .server_address = std::string(server_address), + .port = server_port, + .path = rtsp_path, + .log_level = espp::Logger::Verbosity::WARN, + .accept_task_stack_size_bytes = CONFIG_RTSP_ACCEPT_TASK_STACK_SIZE, + .session_task_stack_size_bytes = CONFIG_RTSP_SESSION_TASK_STACK_SIZE, + .control_task_stack_size_bytes = CONFIG_RTSP_CONTROL_TASK_STACK_SIZE, + }); + server->set_session_log_level(espp::Logger::Verbosity::WARN); + + auto video_packetizer = std::make_shared( + espp::MjpegPacketizer::Config{.max_payload_size = 1400}); + server->add_track( + espp::RtspServer::TrackConfig{.track_id = video_track_id, .packetizer = video_packetizer}); + +#if CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE + auto audio_packetizer = std::make_shared(espp::GenericPacketizer::Config{ + .max_payload_size = 1400, + .payload_type = 97, + .clock_rate = microphone_sample_rate_hz, + .encoding_name = "L16", + .channels = 1, + .fmtp = "", + .media_type = espp::MediaType::AUDIO, + .log_level = espp::Logger::Verbosity::WARN, + }); + server->add_track( + espp::RtspServer::TrackConfig{.track_id = audio_track_id, .packetizer = audio_packetizer}); +#endif + + if (!server->start()) { logger.error("Failed to start RTSP server on {}:{}", server_address, server_port); - rtsp_server.reset(); return false; } // initialize mDNS logger.info("Initializing mDNS"); - esp_err_t err = mdns_init(); + auto err = mdns_init(); if (err != ESP_OK) { logger.error("Could not initialize mDNS: {}", err); - rtsp_server.reset(); + server.reset(); return false; } uint8_t mac[6]; esp_read_mac(mac, ESP_MAC_WIFI_STA); - std::string hostname = fmt::format("camera-streamer-{:x}{:x}{:x}", mac[3], mac[4], mac[5]); + std::string hostname = + fmt::format("{}-{:x}{:x}{:x}", mdns_hostname_prefix, mac[3], mac[4], mac[5]); err = mdns_hostname_set(hostname.c_str()); if (err != ESP_OK) { logger.error("Could not set mDNS hostname: {}", err); mdns_free(); - rtsp_server.reset(); + server.reset(); return false; } logger.info("mDNS hostname set to '{}'", hostname); - err = mdns_instance_name_set("Camera Streamer"); + + err = mdns_instance_name_set(mdns_instance_name); if (err != ESP_OK) { logger.error("Could not set mDNS instance name: {}", err); mdns_free(); - rtsp_server.reset(); + server.reset(); return false; } - err = mdns_service_add("RTSP Server", "_rtsp", "_tcp", server_port, NULL, 0); + + err = mdns_service_add("RTSP Server", "_rtsp", "_tcp", server_port, nullptr, 0); if (err != ESP_OK) { logger.error("Could not add mDNS service: {}", err); mdns_free(); - rtsp_server.reset(); + server.reset(); return false; } + + { + std::lock_guard lock(server_mutex); + rtsp_server = std::move(server); + mdns_started = true; + } + logger.info("mDNS initialized"); return true; } -bool camera_task_fn(std::mutex &m, std::condition_variable &cv) { - auto start = std::chrono::high_resolution_clock::now(); - auto wait_until = [&](auto deadline) { - std::unique_lock lk(m); - cv.wait_until(lk, deadline); - }; +void stop_streaming(void) { + // Move ownership of the tasks / server out from under the lock, then reset + // (which joins the capture tasks) with no lock held. The capture tasks take + // server_mutex every iteration, so joining them while holding it would + // deadlock. + std::unique_ptr old_camera_task; + std::shared_ptr old_rtsp_server; + bool should_stop_mdns = false; +#if CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE + std::unique_ptr old_audio_task; +#endif - // Grab a local reference to the server under the lock, then release it. The - // shared_ptr keeps the server alive for the rest of this iteration even if it - // is reset on another thread, so we never have to hold server_mutex across a - // wait or a send. - std::shared_ptr server; { std::lock_guard lock(server_mutex); - server = rtsp_server; + old_camera_task = std::move(camera_task); + old_rtsp_server = std::move(rtsp_server); + should_stop_mdns = mdns_started; + mdns_started = false; +#if CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE + old_audio_task = std::move(audio_task); +#endif } - if (!server || !server->has_active_sessions()) { - wait_until(start + idle_capture_poll_period); - return false; +#if CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE + old_audio_task.reset(); +#endif + old_camera_task.reset(); + + if (should_stop_mdns) { + logger.info("Deiniting MDNS"); + mdns_free(); } - auto recommended_capture_period = server->get_recommended_capture_period(); - auto capture_cooldown = server->get_capture_cooldown(); - if (capture_cooldown > 0ms) { - wait_until(start + std::max(recommended_capture_period, capture_cooldown)); - return false; + + old_rtsp_server.reset(); +} + +bool camera_task_fn(std::mutex &m, std::condition_variable &cv, bool &task_notified) { + auto start = std::chrono::steady_clock::now(); + + { + std::lock_guard lock(server_mutex); + if (!rtsp_server || !rtsp_server->has_active_sessions()) { + return wait_until_or_stop(m, cv, task_notified, start + idle_capture_poll_period); + } + auto recommended_capture_period = rtsp_server->get_recommended_capture_period(); + auto capture_cooldown = rtsp_server->get_capture_cooldown(); + if (capture_cooldown > 0ms) { + return wait_until_or_stop(m, cv, task_notified, + start + std::max(recommended_capture_period, capture_cooldown)); + } } auto dma_free = heap_caps_get_free_size(MALLOC_CAP_DMA); @@ -362,57 +632,122 @@ bool camera_task_fn(std::mutex &m, std::condition_variable &cv) { dma_free, dma_largest); last_pressure_log = now; } - wait_until(start + dma_pressure_backoff); - return false; + return wait_until_or_stop(m, cv, task_notified, start + dma_pressure_backoff); } - // take image - camera_fb_t *fb = esp_camera_fb_get(); - if (!fb) { + auto *frame_buffer = esp_camera_fb_get(); + if (frame_buffer == nullptr) { logger.error("Camera capture failed"); - return false; + return wait_until_or_stop(m, cv, task_notified, + std::chrono::steady_clock::now() + camera_capture_error_backoff); } - size_t jpg_buf_len = fb->len; - const uint8_t *jpg_buf = fb->buf; + // Drop the frame unless it ends in a valid JPEG EOI marker (0xFF 0xD9). + if (frame_buffer->len < 2 || frame_buffer->buf[frame_buffer->len - 2] != 0xFF || + frame_buffer->buf[frame_buffer->len - 1] != 0xD9) { + static auto last_bad_frame_log = std::chrono::steady_clock::time_point{}; + auto now = std::chrono::steady_clock::now(); + if (now - last_bad_frame_log >= 2s) { + logger.warn("Dropping camera frame with invalid JPEG end marker"); + last_bad_frame_log = now; + } + esp_camera_fb_return(frame_buffer); + return wait_until_or_stop(m, cv, task_notified, + std::chrono::steady_clock::now() + camera_capture_error_backoff); + } - // Drop the frame unless it ends in a valid JPEG EOI marker (0xFF 0xD9); - // a truncated frame would otherwise be packetized and streamed. - if (jpg_buf_len < 2 || jpg_buf[jpg_buf_len - 2] != 0xff || jpg_buf[jpg_buf_len - 1] != 0xd9) { - esp_camera_fb_return(fb); - return false; + std::span jpg_buf(frame_buffer->buf, frame_buffer->len); + { + std::lock_guard lock(server_mutex); + if (rtsp_server) { + rtsp_server->send_frame(video_track_id, jpg_buf); + video_frames_streamed++; + } } - std::span jpg_span(jpg_buf, jpg_buf_len); - server->send_frame(jpg_span); - frames_streamed++; + esp_camera_fb_return(frame_buffer); - esp_camera_fb_return(fb); + // sleep for a short period to target ~10 FPS and yield to other tasks + auto capture_period = target_video_period; + { + std::lock_guard lock(server_mutex); + if (rtsp_server) { + capture_period = std::max(capture_period, rtsp_server->get_recommended_capture_period()); + } + } + return wait_until_or_stop(m, cv, task_notified, start + capture_period); +} - // sleep for a short period to target ~10 FPS to yield to other tasks. - auto capture_period = std::max(target_stream_period, server->get_recommended_capture_period()); - wait_until(start + capture_period); +#if CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE +bool audio_task_fn(std::mutex &m, std::condition_variable &cv, bool &task_notified) { + auto start = std::chrono::steady_clock::now(); - return false; -}; + { + std::lock_guard lock(server_mutex); + if (!rtsp_server || !rtsp_server->has_active_sessions()) { + return wait_until_or_stop(m, cv, task_notified, start + idle_capture_poll_period); + } + auto capture_cooldown = rtsp_server->get_capture_cooldown(); + if (capture_cooldown > 0ms) { + return wait_until_or_stop(m, cv, task_notified, start + capture_cooldown); + } + } -bool memory_monitor_task_fn(std::mutex &m, std::condition_variable &cv, bool &task_notified) { - auto start = std::chrono::high_resolution_clock::now(); - static size_t last_frames_streamed = 0; - auto total_frames = frames_streamed.load(); - auto delta_frames = total_frames - last_frames_streamed; - last_frames_streamed = total_frames; - logger.info("Frames streamed: {} (+{} in last interval)\n{}", total_frames, delta_frames, - espp::HeapMonitor::get_table( - {MALLOC_CAP_DEFAULT, MALLOC_CAP_INTERNAL, MALLOC_CAP_SPIRAM, MALLOC_CAP_DMA})); + if (microphone_rx_channel == nullptr) { + logger.error("Microphone channel is not initialized"); + return wait_until_or_stop(m, cv, task_notified, start + idle_capture_poll_period); + } + + std::array audio_samples{}; + size_t bytes_read = 0; + auto err = i2s_channel_read(microphone_rx_channel, audio_samples.data(), microphone_frame_bytes, + &bytes_read, audio_capture_timeout.count()); + if (err == ESP_ERR_TIMEOUT) { + return false; + } + if (err != ESP_OK) { + logger.error("Failed reading microphone samples: {} '{}'", err, esp_err_to_name(err)); + return wait_until_or_stop(m, cv, task_notified, start + idle_capture_poll_period); + } + if (bytes_read == 0) { + return false; + } + + auto *audio_bytes = reinterpret_cast(audio_samples.data()); { - std::unique_lock lk(m); - auto stop_requested = - cv.wait_until(lk, start + 10s, [&task_notified] { return task_notified; }); - task_notified = false; - if (stop_requested) { - return true; + std::lock_guard lock(server_mutex); + if (rtsp_server) { + std::span audio_frame(audio_bytes, bytes_read); + rtsp_server->send_frame(audio_track_id, audio_frame); + audio_frames_streamed++; } } return false; } +#endif // CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE + +bool memory_monitor_task_fn(std::mutex &m, std::condition_variable &cv, bool &task_notified) { + auto start = std::chrono::steady_clock::now(); + static size_t last_video_frames = 0; + auto total_video_frames = video_frames_streamed.load(); + auto delta_video_frames = total_video_frames - last_video_frames; + last_video_frames = total_video_frames; + +#if CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE + static size_t last_audio_frames = 0; + auto total_audio_frames = audio_frames_streamed.load(); + auto delta_audio_frames = total_audio_frames - last_audio_frames; + last_audio_frames = total_audio_frames; + logger.info("Frames streamed: video={} (+{}), audio={} (+{})\n{}", total_video_frames, + delta_video_frames, total_audio_frames, delta_audio_frames, + espp::HeapMonitor::get_table( + {MALLOC_CAP_DEFAULT, MALLOC_CAP_INTERNAL, MALLOC_CAP_SPIRAM, MALLOC_CAP_DMA})); +#else + logger.info("Frames streamed: {} (+{} in last interval)\n{}", total_video_frames, + delta_video_frames, + espp::HeapMonitor::get_table( + {MALLOC_CAP_DEFAULT, MALLOC_CAP_INTERNAL, MALLOC_CAP_SPIRAM, MALLOC_CAP_DMA})); +#endif + + return wait_until_or_stop(m, cv, task_notified, start + 10s); +} diff --git a/sdkconfig.defaults b/sdkconfig.defaults index 41dc105..a01bf18 100644 --- a/sdkconfig.defaults +++ b/sdkconfig.defaults @@ -1,11 +1,10 @@ -CONFIG_IDF_TARGET="esp32" +# Common defaults shared by all boards. Target-specific settings live in +# sdkconfig.defaults. (esp32 -> TimerCam, esp32s3 -> XIAO ESP32S3 Sense), +# which ESP-IDF loads automatically in addition to this file. CONFIG_COMPILER_OPTIMIZATION_PERF=y # CONFIG_COMPILER_OPTIMIZATION_SIZE=y -CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y -CONFIG_ESPTOOLPY_FLASHSIZE="4MB" - CONFIG_FREERTOS_HZ=1000 # Common ESP-related @@ -13,12 +12,14 @@ CONFIG_FREERTOS_HZ=1000 CONFIG_ESP_SYSTEM_EVENT_TASK_STACK_SIZE=4096 CONFIG_ESP_MAIN_TASK_STACK_SIZE=32768 +# Run at full clock for camera + RTSP workloads +CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y +CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ=240 + # SPIRAM Configuration CONFIG_SPIRAM=y CONFIG_SPIRAM_USE_MALLOC=y -CONFIG_SPIRAM_MODE_OCT=y CONFIG_SPIRAM_SPEED_80M=y -# CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY=y # # Partition Table @@ -26,27 +27,13 @@ CONFIG_SPIRAM_SPEED_80M=y CONFIG_PARTITION_TABLE_CUSTOM=y CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" - -# ESP32-specific -# -CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y -CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ=240 - -# ESP32-Camera specific -# CONFIG_SCCB_HARDWARE_I2C_DRIVER_LEGACY=y - -# CONFIG_LWIP_IRAM_OPTIMIZATION=y +# Networking tuning for higher-throughput streaming CONFIG_LWIP_TCPIP_TASK_PRIO=23 - -# -# ESP32S3-specific -# CONFIG_ESP_WIFI_STATIC_RX_BUFFER_NUM=8 CONFIG_ESP_WIFI_DYNAMIC_RX_BUFFER_NUM=16 CONFIG_ESP_WIFI_STATIC_TX_BUFFER=y CONFIG_ESP_WIFI_STATIC_TX_BUFFER_NUM=16 CONFIG_ESP_WIFI_AMPDU_TX_ENABLED=y - CONFIG_LWIP_TCP_SND_BUF_DEFAULT=16384 CONFIG_LWIP_TCP_WND_DEFAULT=16384 CONFIG_LWIP_TCP_RECVMBOX_SIZE=16 diff --git a/sdkconfig.defaults.esp32 b/sdkconfig.defaults.esp32 new file mode 100644 index 0000000..79708fb --- /dev/null +++ b/sdkconfig.defaults.esp32 @@ -0,0 +1,9 @@ +# M5Stack ESP32 TimerCam (OV3660). Loaded automatically by ESP-IDF in addition +# to sdkconfig.defaults when the target is esp32. +CONFIG_IDF_TARGET="esp32" + +CONFIG_CAMERA_STREAMER_BOARD_ESP32_TIMER_CAM=y + +# 4MB flash +CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y +CONFIG_ESPTOOLPY_FLASHSIZE="4MB" diff --git a/sdkconfig.defaults.esp32s3 b/sdkconfig.defaults.esp32s3 new file mode 100644 index 0000000..8020437 --- /dev/null +++ b/sdkconfig.defaults.esp32s3 @@ -0,0 +1,26 @@ +# Seeed Studio XIAO ESP32S3 Sense (OV2640 camera + PDM microphone). Loaded +# automatically by ESP-IDF in addition to sdkconfig.defaults when the target is +# esp32s3. +CONFIG_IDF_TARGET="esp32s3" + +CONFIG_CAMERA_STREAMER_BOARD_XIAO_ESP32S3_SENSE=y + +# 8MB flash +CONFIG_ESPTOOLPY_FLASHSIZE_8MB=y +CONFIG_ESPTOOLPY_FLASHSIZE="8MB" +CONFIG_ESPTOOLPY_FLASHMODE_QIO=y + +# The ESP32-S3 has native USB; route the console to USB Serial/JTAG so the CLI +# works over the single USB-C connector. +CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y + +# Octal PSRAM on the ESP32-S3R8 module +CONFIG_SPIRAM_MODE_OCT=y + +# Give esp-timer headroom for the camera + RTSP workload +CONFIG_ESP_TIMER_TASK_STACK_SIZE=6144 + +# RTSP task stack sizing (video + audio multitrack) +CONFIG_RTSP_ACCEPT_TASK_STACK_SIZE=4096 +CONFIG_RTSP_SESSION_TASK_STACK_SIZE=4096 +CONFIG_RTSP_CONTROL_TASK_STACK_SIZE=8192