Merge pull request 'dev' (#1) from dev into main

Reviewed-on: https://forge.chapril.org/UncleSamulus/BirdNET-stream/pulls/1

.dockerignore

@@ -0,0 +1,7 @@
+/var
+/.venv
+/.github
+/.ideas
+/media
+/daemon/systemd
+/analyzer

.env

@@ -1 +0,0 @@
-CUDA_VISIBLE_DEVICES=""

.env.example

@@ -0,0 +1,12 @@
+# BirdNET-Analyzer environment
+CUDA_VISIBLE_DEVICES=""
+
+# BirdNET-stream environment
+DATABASE_USER="birdnet"
+DATABASE_PASSWORD="secret" # change this
+DATABASE_PORT="3306"
+DATABASE_ROOT_PASSWORD="secret" # change this
+
+RECORDS_DIR="/media/data/birdnet/records"
+CHARTS_DIR="/media/data/birdnet/charts"
+SERVER_NAME="birdnet.local"

.gitignore

@@ -1,13 +1,17 @@
 var/
 /.venv/
-/analyzer/
 
-.env
+/.env
+/.env.local
+!.env.local.example
+!.env.example
 
 species_list.txt
 
 push.sh
 
-config/*.conf
+/config/*.conf
-
+!/config/*.conf.example
+/config/stations/*.conf
+!/config/stations/*.conf.example
 .vscode/

.ideas/birdnet_archive.sh

@@ -0,0 +1,133 @@
+#! /usr/bin/env bash
+# Compress wav to flac and archive them as zip
+
+# Requires: tar, gzip, ffmpeg
+
+set -e
+
+DEBUG=${DEBUG:-0}
+debug() {
+    [[ $DEBUG -eq 1 ]] && echo "$@"
+}
+error() {
+    echo 1>&2 "$@"
+}
+
+audio_compress() {
+    local filepath
+    filepath="$1"
+    if [[ "$DRY" -eq 1 ]]; then
+        debug "Would compress $filepath to flac"
+        return 0
+    else
+        debug "Compressing $filepath"
+        ffmpeg -i "$filepath" -acodec flac -compression_level 10 "${filepath%.wav}.flac"
+    fi
+}
+
+all_audio_compress() {
+    local dir
+    dir="$1"
+    debug "Compressing all .wav audio in $dir"
+    for filepath in "$dir"/*.wav; do
+        if [[ "$DRY" -eq 1 ]]; then
+            debug "Would convert $filepath to flac and remove it"
+        else
+            audio_compress "$filepath"
+            debug "Removing $filepath"
+            rm "$filepath"
+        fi
+    done
+}
+
+record_datetime() {
+    source_wav=$1
+    source_base=$(basename "$source_wav" ".wav")
+    record_date=$(echo "$source_base" | cut -d"_" -f2)
+    record_time=$(echo "$source_base" | cut -d"_" -f3)
+    YYYY=$(echo "$record_date" | cut -c 1-4)
+    MM=$(echo "$record_date" | cut -c 5-6)
+    DD=$(echo "$record_date" | cut -c 7-8)
+    HH=$(echo "$record_time" | cut -c 1-2)
+    MI=$(echo "$record_time" | cut -c 3-4)
+    SS=$(echo "$record_time" | cut -c 5-6)
+    SSS="000"
+    date="$YYYY-$MM-$DD $HH:$MI:$SS.$SSS"
+    echo "$date"
+}
+
+source_wav() {
+    model_output_dir="$1"
+    wav=$(basename "$model_output_dir" | rev | cut --complement -d"." -f1 | rev)
+    echo "$wav"
+}
+
+birdnet_archive_older_than() {
+    local days
+    days="$1"
+    local date
+    date=$(date +"%Y-%m-%d")
+    local date_pivot
+    date_pivot=$(date -d "$date + $days days" +"%Y-%m-%d")
+    move_records_to_archive "$date_pivot"
+    zip_archives
+}
+
+move_records_to_archive() {
+    local date
+    date="$1"
+    local archives_dir
+    archives_dir="$2"
+    archive_path="${ARCHIVE_DIR}/$date"
+    debug "Moving records from $CHUNK_FOLDER/out to $archives_path"
+    for filepath in $(find "$CHUNK_FOLDER/out/" -name '*.wav.d'); do
+        wav=$(source_wav "$filepath")
+        dir=$(dirname "$filepath")
+        record_datetime=$(record_datetime "$wav")
+        if [[ "$record_datetime" == "$date" ]]; then
+            debug "Moving $filepath to $archive_path"
+            if [[ ! -d "$archive_path" ]]; then
+                mkdir -p "$archive_path"
+            fi
+            mv "$filepath" "$archive_path"
+            debug "Moving model output directory to archive"
+            mv "$dir" "$archive_path/"
+            debug "Moving wav to archive"
+            mv "$CHUNK_FOLDER/out/$wav" "$archive_path/"
+        fi
+    done
+}
+
+zip_archives() {
+    debug "Zipping archives in ${ARCHIVE_DIR}"
+    for archive_path in $(find "${ARCHIVE_DIR}" -type d); do
+        archive_name="birdnet_$(basename "$archive_path" | tr '-' '').tar.gz"
+        if [[ "$DRY" -eq 1 ]]; then
+            debug "Would zip $archive_path to $archive_name"
+        else
+            debug "Zipping $archive_path to $archive_name"
+            tar -czf "$archive_name" -C "$archive_path" .
+            debug "Removing temporary archive folder in ${ARCHIVE_DIR}"
+            rm -rf "$archive_path"
+        fi
+    done
+}
+
+main() {
+    config_filepath="./config/birdnet.conf"
+    [ -f "$config_filepath" ] || {
+        error "Config file not found: $config_filepath"
+        exit 1
+    }
+    source "$config_filepath"
+    if [[ -z "CHUNK_FOLDER" ]]; then
+        error "CHUNK_FOLDER not set in config file"
+        exit 1
+    fi
+    if [[ -z "ARCHIVE_FOLDER" ]]; then
+        error "ARCHIVE_FOLDER not set in config file"
+        exit 1
+    fi
+    debug "Launch birdnet archive script from $CHUNK_FOLDER to $ARCHIVE_FOLDER"
+    birdnet_archive_older_than $DAYS_TO_KEEP
+}

.ideas/database.py

@@ -7,7 +7,7 @@ verbose = False
 
 """Load config"""
 def load_conf():
-    with open("./config/analyzer.conf", "r") as f:
+    with open("./config/birdnet.conf", "r") as f:
         conf = f.readlines()
         res = dict(map(str.strip, sub.split('=', 1)) for sub in conf if '=' in sub)
     return res

.ideas/fill_taxa.sh

@@ -3,7 +3,7 @@
 set -e
 
 # Load config file
-config_filepath="./config/analyzer.conf"
+config_filepath="./config/birdnet.conf"
 
 if [ -f "$config_filepath" ]; then
     source "$config_filepath"

.ideas/insert_observation.sh

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 # Load config file
-config_filepath="./config/analyzer.conf"
+config_filepath="./config/birdnet.conf"
 
 if [ -f "$config_filepath" ]; then
     source "$config_filepath"

CHANGELOG.md

@@ -1,6 +1,11 @@
 # Changelog
+## v0.0.1-alpha
 
-## v0.0.1-rc
+- Add docker compose port
+- Improve install script
+- Add base uninstall script (need deeper work)
+- Add ttyd for systemd logging
+## v0.0.1-rc (2022-08-18)
 
 - Integrate BirdNET-Analyzer as submodule
 - Add birdnet_recording service

INSTALL.md

@@ -8,6 +8,18 @@ For a one-liner installation, you can use the following command:
 curl -sL https://raw.githubusercontent.com/UncleSamulus/BirdNET-stream/main/install.sh | bash
 ```
 
+For debug purposes, you can use the following command, it will log the installation steps to the console:
+
+```bash
+DEBUG=1 ./install.sh
+```
+
+If you need to use a specific branch (e.g. dev), you can use the following command:
+
+```bash
+BRANCH=dev ./install.sh
+```
+
 ## Requirements
 
 - git
@@ -21,7 +33,7 @@ curl -sL https://raw.githubusercontent.com/UncleSamulus/BirdNET-stream/main/inst
 
 ```bash
 sudo apt-get update
-sudo apt-get install python3-dev python3-pip
+sudo apt-get install python3-dev python3-pip python3-venv
 sudo pip3 install --upgrade pip
 ```
 
@@ -67,17 +79,23 @@ sudo systemctl enable --now birdnet_recording.service birdnet_analyzis.service b
 #### Check if services are working
 
 ```bash
-# Sercices status
+# Sercices and timers status
-sudo systemctl status birdnet_recording.service birdnet_analyzis.service
+sudo systemctl status birdnet_\*
-# Timers status
-sudo systemctl status birdnet_miner.timer
 ```
 
 ```bash
 # BirdNET-stream logs
-sudo journalctl -feu {birdnet_recording,birdnet_analyzis}.service
+sudo journalctl -feu birdnet_\*
 ```
 
+#### Enable `loginctl-linger` for the user that runs the servuces
+
+Running:
+```bash
+loginctl enable-linger
+```
+This allows to use `/run/user/1000/pulse` to record audio using PulseAudio in birdnet_recording.sh.
+
 ## Setup BirdNET-stream symfony webapp
 
 ### Install php 8.1
@@ -111,7 +129,7 @@ sudo mv /composer.phar /usr/local/bin/composer
 
 ```bash
 cd www
-composer install
+composer install --no-dev --prefer-dist --optimize-autoloader
 ```
 
 ### Install nodejs and npm
@@ -129,7 +147,7 @@ nvm use 16
 ```
 
 ```bash
-sudo dnf install npm
+sudo apt-get install npm
 ```
 
 ```bash
@@ -170,7 +188,7 @@ Launch and enable icecast:
 sudo systemctl enable --now icecast2
 ```
 
-Adapt `config/analyzer.conf` to this configuration:
+Adapt `config/birdnet.conf` to this configuration:
 
 ```conf
 ICECAST_USER=source
@@ -260,3 +278,28 @@ sudo crontab -e
 ```
 
 (This updates the certicates every first day of the month, feel free to adapt to your needs.)
+
+## Setup ttyd to stream audio to webapp
+
+Change to a dedicated folder, build and install ttyd:
+
+```bash
+cd /opt
+sudo wget wget https://github.com/tsl0922/ttyd/releases/download/1.7.1/ttyd.x86_64 # Change to your architecture and get last version
+sudo mv ttyd.x86_64 ttyd
+sudo chmod +x ttyd
+```
+
+Set up birdnet_ttyd systemd service to start as a daemon:
+
+```bash
+# Copy service template
+sudo cp ./daemon/systemd/templates/birdnet_ttyd.service /etc/systemd/system/birdnet_ttyd.service
+# Edit template and adapt placeholders
+sudo vim /etc/systemd/system/birdnet_ttyd.service
+# Enable and start ttyd service
+sudo systemctl daemon-reload
+sudo systemctl enable --now birdnet_ttyd.service
+```
+
+Then go to [https://birdnet.lan/ttyd](https://birdnet.lan/ttyd) and start streaming logs.

README.md

@@ -28,13 +28,39 @@ It should work on a Raspberry Pi (or other Single Board Computer) with a USB mic
 
 ## Installation
 
-On debian based system, you can install BirdNET-stream with the following command:
+> **Warning** BirdNET-stream is in early development, and may not work properly...
+
+<!-- On debian based system, you can install BirdNET-stream with the following command:
 
 ```bash
 curl -sL https://raw.githubusercontent.com/UncleSamulus/BirdNET-stream/main/install.sh | bash
+``` -->
+
+On debian based systems (tested on Debian Bullseye), the following command should allow you to install the base components without too much trouble:
+
+```bash
+# Change to your installation directory here, /home/$USER/Documents/BirdNET-stream for instance, or /opt/birdnet-stream, or whatever
+cd /path/to/installation/directory
+# Download installation script 
+curl -0 https://raw.githubusercontent.com/UncleSamulus/BirdNET-stream/main/install.sh
+# Run installation script:
+chmod +x ./install.sh
+./install.sh
 ```
 
-For finer control, or to adapt to your system, you can follow the instructions in the [INSTALL.md](./INSTALL.md) file.
+I recommend to add `DEBUG=1` before this command to see the installation steps:
+```bash
+DEBUG=1 ./install.sh
+```
+
+To install from a specific git branch, add `BRANCH=<branch>` before the command, for instance:
+
+```bash
+BRANCH=dev DEBUG=1 ./install.sh
+```
+
+For finer control, or to adapt to your system, you can follow the instructions in the [INSTALL.md](./INSTALL.md) file (it may unfortunatly not be accurate for your system).
+
 
 ## Usage
 

TODO

@@ -1,5 +1,4 @@
-- Fix clean script
 - Fix service manager
-- Add docker support
 - Species i18n
 - File purge policy
+- Add and test RTSP support

config/birdnet.conf.example

@@ -16,8 +16,7 @@ CHUNK_FOLDER="./var/chunks"
 AUDIO_DEVICE="default"
 # Virtual env for BirdNET AI with required packages
 PYTHON_VENV="./.venv/birdnet-stream"
+WORKDIR="/home/$USER/BirdNET-stream"
 # Database location
 DATABASE="./var/db.sqlite"
-
+# DATABASE="mysql://birdnet:secret@localhost:3306/birdnet_observations" # uncomment and change 'secret' if you want to use a mariadb (mysql) database instea of sqlite
-DAEMON_USER="birdnet"
-DAEMON_PASSWORD="secret"

config/stations/station.conf.example

@@ -0,0 +1,3 @@
+# Config file for a distant recording station 
+RTSP_URL=rtsp://host:1000/birdnet/stream
+STATION_NAME=garden

daemon/birdnet_analyzis.sh

@@ -3,12 +3,10 @@ set -e
 
 DEBUG=${DEBUG:-1}
 debug() {
-    if [ $DEBUG -eq 1 ]; then
+    [[ $DEBUG -eq 1 ]] && echo "$@"
-        echo "$1"
-    fi
 }
 
-config_filepath="./config/analyzer.conf"
+config_filepath="./config/birdnet.conf"
 
 if [ -f "$config_filepath" ]; then
     source "$config_filepath"
@@ -64,7 +62,9 @@ check_prerequisites() {
 
 # Get array of audio chunks to be processed
 get_chunk_list() {
-    find "${CHUNK_FOLDER}/in" -type f -name '*.wav' -exec basename {} \; ! -size 0 | sort
+    chunk_list=($(ls ${CHUNK_FOLDER}/in))
+    echo "${chunk_list}"
+    # find "${CHUNK_FOLDER}/in" -type f -name '*.wav' -exec basename {} \; ! -size 0 | sort
 }
 
 # Perform audio chunk analysis on one chunk
@@ -75,13 +75,22 @@ analyze_chunk() {
     mkdir -p "$output_dir"
     date=$(echo $chunk_name | cut -d'_' -f2)
     week=$(./daemon/weekof.sh $date)
-    $PYTHON_EXECUTABLE ./analyzer/analyze.py --i $chunk_path --o "$output_dir/model.out.csv" --lat $LATITUDE --lon $LONGITUDE --week $week --min_conf $CONFIDENCE --threads 4 --rtype csv
+    if [[ ! -z "${THREADS}" ]]; then
+        threads="--threads ${THREADS}"
+    else 
+        threads=""
+    fi
+    $PYTHON_EXECUTABLE ./analyzer/analyze.py --i $chunk_path --o "$output_dir/model.out.csv" --lat $LATITUDE --lon $LONGITUDE --week $week --min_conf $CONFIDENCE $threads --rtype csv
     debug "Model output written to $output_dir/model.out.csv"
+    bash ./daemon/birdnet_output_to_sql.sh "$output_dir/model.out.csv"
+    debug "Dumped to SQL database"
 }
 
 # Perform audio chunk analysis on all recorded chunks
 analyze_chunks() {
-    for chunk_name in $(get_chunk_list); do
+    local chunks
+    chunks="${1}"
+    for chunk_name in "${chunks}"; do
         if [[ -f "${CHUNK_FOLDER}/out/$chunk_name.d/model.out.csv" ]]; then
             debug "Skipping $chunk_name, as it has already been analyzed"
         else
@@ -98,4 +107,4 @@ check_prerequisites
 chunks=$(get_chunk_list)
 
 # Analyze all chunks in working directory
-analyze_chunks $chunks
+analyze_chunks "$chunks"

daemon/birdnet_clean.sh

@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 ##
-## Clean up var folder from useless files
+## Clean up var folder from useless files (e.g empty wav, audio with no bird, etc)
 ##
 
 set -e
@@ -13,7 +13,7 @@ debug() {
     fi
 }
 
-config_filepath="./config/analyzer.conf"
+config_filepath="./config/birdnet.conf"
 
 if [ -f "$config_filepath" ]; then
     source "$config_filepath"

daemon/birdnet_manager.sh

@@ -3,7 +3,7 @@
 set -e 
 # set -x
 
-config_filepath="./config/analyzer.conf"
+config_filepath="./config/birdnet.conf"
 
 if [ -f "$config_filepath" ]; then
     source "$config_filepath"

daemon/birdnet_output_to_sql.sh

@@ -1,33 +1,21 @@
 #! /usr/bin/env bash
-# Extract observations from a model output folder
+# Extract observations from a model output file into SQL database
 #
 
 DEBUG=${DEBUG:-1}
 set -e
 # set -x
-
+DEBUG=${DEBUG:-1}
 debug() {
-    if [ $DEBUG -eq 1 ]; then
+    [[ $DEBUG -eq 1 ]] && echo "$@"
-        echo "$1"
-    fi
 }
 
 # Load bash library to deal with BirdNET-stream database
 source ./daemon/database/scripts/database.sh
 
 # Load config
-source ./config/analyzer.conf
+source ./config/birdnet.conf
 # Check config
-if [[ -z ${CHUNK_FOLDER} ]]; then
-    echo "CHUNK_FOLDER is not set"
-    exit 1
-else
-    if [[ ! -d ${CHUNK_FOLDER}/out ]]; then
-        echo "CHUNK_FOLDER does not exist: ${CHUNK_FOLDER}/out"
-        echo "Cannot extract observations."
-        exit 1
-    fi
-fi
 
 if [[ -z ${LATITUDE} ]]; then
     echo "LATITUDE is not set"
@@ -39,10 +27,6 @@ if [[ -z ${LONGITUDE} ]]; then
     exit 1
 fi
 
-model_outputs() {
-    ls ${CHUNK_FOLDER}/out/*/model.out.csv
-}
-
 source_wav() {
     model_output_path=$1
     model_output_dir=$(dirname $model_output_path)
@@ -107,13 +91,6 @@ save_observations() {
     done
 }
 
-main() {
+model_output_path="$1"
-    # # Remove all junk observations
-    # ./daemon/birdnet_clean.sh
-    # Get model outputs
-    for model_output in $(model_outputs); do
-        save_observations $model_output
-    done
-}
 
-main
+save_observations $model_output_path

daemon/birdnet_purge.sh

@@ -0,0 +1,145 @@
+#! /usr/bin/env bash
+# Remove as much as possible as audio files that are not really helpful
+#
+
+set -e
+# set -x
+
+DEBUG=${DEBUG:-0}
+DRY=${DRY:-0}
+debug() {
+    [[ $DEBUG -eq 1 ]] && echo "$@"
+}
+
+remove_selected() {
+    local selected_audios
+    selected_audios="$1"
+    for audio in $selected_audios; do
+        debug "Removing $audio"
+        rm "$CHUNK_FOLDER/out/$audio"
+    done
+}
+
+remove_audios_older_than() {
+    local pivot_date=$1
+    touch -t "$(date -d "$pivot_date" +"%Y%m%d%H%M")" /tmp/birdnet_purge.sh.pivot_date
+    if [[ $DRY -eq 1 ]]; then
+        find "$CHUNK_FOLDER/out/" -type f -name '*.wav' -not -newer /tmp/birdnet_purge.sh.pivot_date
+    else
+        find "$CHUNK_FOLDER/out/" -type f -name '*.wav' -not -newer /tmp/birdnet_purge.sh.pivot_date -delete
+    fi
+}
+
+# Remove audios containing only excluded species
+remove_audios_containing_only_excluded_species() {
+    local excluded_species
+    excluded_species=$1
+    local audios
+    audios=$(find "$CHUNK_FOLDER/out/" -type f -name '*.wav')
+    audios=$(only_audios_containing_excluded_species "$audios" "$excluded_species")
+    if [[ $DRY -eq 1 ]]; then
+        echo "$audios"
+    else
+        remove_selected "$audios"
+    fi
+}
+
+# Filter audio list, keep only those that contains only bird calls of selected exclude list
+only_audios_containing_excluded_species() {
+    local audios
+    audios=$1
+    local excluded_species
+    excluded_species=$2
+    local selected
+    selected=""
+    if [[ -z $excluded_specie ]]; then
+        echo "No species to exclude"
+        return 1
+    fi
+    for file in $audios; do
+        if [[ $(contains_only_excluded_species "$file" "$excluded_species") -eq 1 ]]; then
+            selected="$selected $file"
+        fi
+    done
+}
+
+# Check whether the audio file contains only excluded species
+contains_only_excluded_species() {
+    local audio
+    audio=$1
+    local excluded_species
+    excluded_species=$2
+    local flag
+    flag=1
+    IFS=$','
+    local regex
+    for species in $(get_contacted_species "$audio"); do
+        regex="$species"
+        if [[ $excluded_species =~ $regex ]]; then
+            flag=0
+            break
+        fi
+    done
+    echo flag
+}
+
+# Get all scientific names of species detected by the model
+get_contacted_species() {
+    local audio
+    audio=$1
+    local model_output_path
+    model_output_path="$CHUNK_FOLDER/out/$audio.d/model.out.csv"
+    observations=$(tail -n +2 < "$model_output_path")
+    IFS=$'\n'
+    debug "Observations retrieved from $model_output_path"
+    local species
+    local contacted_species
+    contacted_species=""
+    for observation in $observations; do
+        if [[ -z "$observation" ]]; then
+            continue
+        fi
+        species=$(echo "$observation" | cut -d"," -f3)
+        contacted_species="${contacted_species},${species}"
+    done
+    echo "$contacted_species"
+}
+
+main() {
+    debug "Launching birdnet purge script"
+    local config_path
+    config_path="./config/birdnet.conf"
+    if [[ ! -f $config_path ]]; then
+        echo "Config file $config_path not found"
+        exit 1
+    fi
+    source "$config_path"
+    if [[ -z ${CHUNK_FOLDER} ]]; then
+        echo "CHUNK_FOLDER is not set"
+        exit 1
+    else
+        if [[ ! -d ${CHUNK_FOLDER}/out ]]; then
+            echo "CHUNK_FOLDER does not exist: ${CHUNK_FOLDER}/out"
+            echo "Cannot clean absent folder."
+            exit 1
+        fi
+    fi
+    today=$(date +"%Y-%m-%d")
+    pivot_date=$(date -d "$today - $DAYS_TO_KEEP days" +"%Y-%m-%d")
+    debug "Recordings older than $pivot_date will be removed"
+    remove_audios_older_than "$pivot_date"
+    if [[ -z ${EXCLUDED_SPECIES} ]]; then
+        echo "No species to exclude"
+        exit 1
+    else
+        if [[ -f ${EXCLUDED_SPECIES} ]]; then
+            excluded_species=$(cat "${EXCLUDED_SPECIES}")
+            remove_audios_containing_only_excluded_species "$excluded_species"
+        else
+            echo "Excluded species file ${EXCLUDED_SPECIES} not found"
+            exit 1
+        fi
+    fi
+}
+
+main

daemon/birdnet_recording.sh

@@ -1,8 +1,9 @@
 #! /usr/bin/env bash
 
-DEBUG=${DEBUG:-0}
+DEBUG=${DEBUG:-1}
 
 export PULSE_RUNTIME_PATH="/run/user/$(id -u)/pulse/"
+FFMPEG_OPTIONS="-nostdin -hide_banner -loglevel error -nostats -vn -acodec pcm_s16le -ac 1 -ar 48000"
 
 debug() {
     if [ $DEBUG -eq 1 ]; then
@@ -23,19 +24,32 @@ record_loop() {
     DURATION=$2
     debug "New recording loop."
     while true; do
-        record $DEVICE $DURATION
+        record_device $DEVICE $DURATION
     done
 }
 
 
-record() {
+record_stream() {
+    local STREAM=$1
+    local DURATION=$2
+    local debug "Recording from $STREAM for $DURATION seconds"
+    ffmpeg $FFMPEG_OPTIONS -i ${STREAM} -t ${DURATION} file:${CHUNK_FOLDER}/in/birdnet_$(date "+%Y%m%d_%H%M%S").wav
+}
+
+record_device() {
     DEVICE=$1
     DURATION=$2
     debug "Recording from $DEVICE for $DURATION seconds"
-    ffmpeg -nostdin -hide_banner -loglevel error -nostats -f pulse -i ${DEVICE} -t ${DURATION} -vn -acodec pcm_s16le -ac 1 -ar 48000 -af "volume=$RECORDING_AMPLIFY" file:${CHUNK_FOLDER}/in/birdnet_$(date "+%Y%m%d_%H%M%S").wav
+    local ffmpeg_input
+    if [[ "$AUDIO_USE_PULSE" = "true" ]]; then
+        ffmpeg_input="-f pulse -i ${DEVICE}"
+    else
+        ffmpeg_input="-f alsa -i ${DEVICE}"
+    fi
+    ffmpeg $FFMPEG_OPTIONS $ffmpeg_input -t ${DURATION} -af "volume=$RECORDING_AMPLIFY" file:${CHUNK_FOLDER}/in/birdnet_$(date "+%Y%m%d_%H%M%S").wav
 }
 
-config_filepath="./config/analyzer.conf"
+config_filepath="./config/birdnet.conf"
 
 if [ -f "$config_filepath" ]; then
     source "$config_filepath"
@@ -48,9 +62,18 @@ check_folder
 
 [ -z $RECORDING_DURATION ] && RECORDING_DURATION=15
 
-if [[ -z $AUDIO_DEVICE ]]; then
+if [[ $AUDIO_RECORDING = "true" ]]; then
-    echo "AUDIO_DEVICE is not set"
+    debug "Recording with on board device"
-    exit 1
+    if [[ -z $AUDIO_DEVICE ]]; then
+        echo "AUDIO_DEVICE is not set"
+        exit 1
+    fi
+    record_loop $AUDIO_DEVICE $RECORDING_DURATION
 fi
 
-record_loop $AUDIO_DEVICE $RECORDING_DURATION
+if [[ $AUDIO_STATIONS = "true" ]]; then
+    for station in $(ls ./config/stations/*.conf); do
+        source $station
+        record_stream $STATION_URL $RECORDING_DURATION
+    done
+fi

daemon/birdnet_streaming.sh

@@ -18,7 +18,7 @@ stream() {
         -f mp3 "icecast://source:${ICECAST_PASSWORD}@${ICECAST_HOST}:${ICECAST_PORT}/${ICECAST_MOUNT}" -listen 1
 }
 
-config_filepath="./config/analyzer.conf"
+config_filepath="./config/birdnet.conf"
 
 if [ -f "$config_filepath" ]; then
     source "$config_filepath"

daemon/database/migrations/migrate.sh

@@ -8,7 +8,7 @@ debug() {
 }
 
 # Load config file
-config_filepath="./config/analyzer.conf"
+config_filepath="./config/birdnet.conf"
 
 if [ -f "$config_filepath" ]; then
     source "$config_filepath"

daemon/database/scripts/create.sh

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 # Load config file
-config_filepath="./config/analyzer.conf"
+config_filepath="./config/birdnet.conf"
 
 if [ -f "$config_filepath" ]; then
     source "$config_filepath"
@@ -17,5 +17,16 @@ if [ -z "$DATABASE" ]; then
     DATABASE="./var/db.sqlite"
 fi
 
-# Create database according to schema in structure.sql
+if [[ $DATABASE = "mysql://"* ]]; then
-sqlite3 "$DATABASE" < ./daemon/database/structure.sql
+    # Split mysql uri into user, password, host, port, and database
+    MYSQL_ADDRESS=$(echo "$DATABASE" | sed 's/mysql:\/\///g')
+    MYSQL_CREDENTIALS=$(echo "$MYSQL_ADDRESS" | cut -d@ -f1)
+    MYSQL_USER=$(echo "$MYSQL_CREDENTIALS" | cut -d: -f1)
+    MYSQL_PASSWORD=$(echo "$MYSQL_CREDENTIALS" | cut -d: -f2)
+    MYSQL_HOST=$(echo "$MYSQL_ADDRESS" | cut -d@ -f2 | cut -d: -f1)
+    MYSQL_PORT=$(echo "$MYSQL_ADDRESS" | cut -d@ -f2 | cut -d: -f2 | cut -d/ -f1)
+    MYSQL_DATABASE=$(echo "$MYSQL_ADDRESS" | cut -d/ -f2)
+    mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -h$MYSQL_HOST -P$MYSQL_PORT -D$MYSQL_DATABASE < ./daemon/database/structure-mysql.sql
+else
+    sqlite3 $DATABASE < ./daemon/database/structure-sqlite.sql
+fi

daemon/database/scripts/database.sh

@@ -1,36 +1,60 @@
 #! /usr/bin/env bash
-# SQLite library to deal with BirdNET-stream database
+# SQLite library to deal with BirdNET-stream observations database
 
 set -e
 
-source ./config/analyzer.conf
+source ./config/birdnet.conf
 
 # Create database in case it was not created yet
 ./daemon/database/scripts/create.sh
 
-DATABASE=${DATABASE:-"./var/db.sqlite"}
+# Check if database location is specified
+if [ -z "$DATABASE" ]; then
+    echo "DATABASE location not specified"
+    echo "Defaults to ./var/db.sqlite"
+    DATABASE="./var/db.sqlite"
+fi
+
+query() {
+    local stmt
+    stmt="$1"
+    if [[ $DATABASE = "mysql://"* ]]; then
+        # Split mysql uri into user, password, host, port, and database
+        MYSQL_ADDRESS=$(echo "$DATABASE" | sed 's/mysql:\/\///g')
+        MYSQL_CREDENTIALS=$(echo "$MYSQL_ADDRESS" | cut -d@ -f1)
+        MYSQL_USER=$(echo "$MYSQL_CREDENTIALS" | cut -d: -f1)
+        MYSQL_PASSWORD=$(echo "$MYSQL_CREDENTIALS" | cut -d: -f2)
+        MYSQL_HOST=$(echo "$MYSQL_ADDRESS" | cut -d@ -f2 | cut -d: -f1)
+        MYSQL_PORT=$(echo "$MYSQL_ADDRESS" | cut -d@ -f2 | cut -d: -f2 | cut -d/ -f1)
+        MYSQL_DATABASE=$(echo "$MYSQL_ADDRESS" | cut -d/ -f2)
+        mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -h$MYSQL_HOST -P$MYSQL_PORT -D$MYSQL_DATABASE -e "$stmt"
+    else
+        sqlite3 -cmd ".timeout 1000" "$DATABASE" "$stmt"
+    fi
+
+}
 
 get_location_id() {
-    sqlite3 -cmd ".timeout 1000" $DATABASE "SELECT location_id FROM location WHERE latitude=$1 AND longitude=$2"
+    query "SELECT location_id FROM location WHERE latitude=$1 AND longitude=$2"
 }
 
 get_taxon_id() {
-    sqlite3 -cmd ".timeout 1000" $DATABASE "SELECT taxon_id FROM taxon WHERE scientific_name='$1'"
+    query "SELECT taxon_id FROM taxon WHERE scientific_name='$1'"
 }
 
 insert_taxon() {
-    sqlite3 -cmd ".timeout 1000" $DATABASE "INSERT INTO taxon (scientific_name, common_name) VALUES (\"$1\", \"$2\")"
+    query "INSERT INTO taxon (scientific_name, common_name) VALUES (\"$1\", \"$2\")"
 }
 
 insert_location() {
-    sqlite3 -cmd ".timeout 1000" $DATABASE "INSERT INTO location (latitude, longitude) VALUES ($1, $2)"
+    query "INSERT INTO location (latitude, longitude) VALUES ($1, $2)"
 }
 
 insert_observation() {
-    sqlite3 -cmd ".timeout 1000" $DATABASE "INSERT INTO observation (audio_file, start, end, taxon_id, location_id, confidence, date) VALUES ('$1', '$2', '$3', '$4', '$5', '$6', '$7')"
+    query "INSERT INTO observation (audio_file, start, end, taxon_id, location_id, confidence, date) VALUES ('$1', '$2', '$3', '$4', '$5', '$6', '$7')"
 }
 
 # Check if the observation already exists in the database
 observation_exists() {
-    sqlite3 -cmd ".timeout 1000" $DATABASE "SELECT EXISTS(SELECT observation_id FROM observation WHERE audio_file='$1' AND start='$2' AND end='$3' AND taxon_id='$4' AND location_id='$5')"
+    query "SELECT EXISTS(SELECT observation_id FROM observation WHERE audio_file='$1' AND start='$2' AND end='$3' AND taxon_id='$4' AND location_id='$5')"
 }

daemon/database/structure-mysql.sql

@@ -0,0 +1,31 @@
+/** Database structure for BirdNET-stream SQLite*/
+
+/** Taxon table */
+CREATE TABLE IF NOT EXISTS taxon (
+    taxon_id INTEGER PRIMARY KEY NOT NULL AUTO_INCREMENT,
+    scientific_name TEXT NOT NULL,
+    common_name TEXT NOT NULL
+);
+
+/** Location table */
+CREATE TABLE IF NOT EXISTS location (
+    location_id INTEGER PRIMARY KEY NOT NULL AUTO_INCREMENT,
+    latitude REAL NOT NULL,
+    longitude REAL NOT NULL
+);
+
+/** Observation table */
+CREATE TABLE IF NOT EXISTS observation (
+    `observation_id` INTEGER PRIMARY KEY NOT NULL AUTO_INCREMENT,
+    `audio_file` TEXT NOT NULL,
+    `start` REAL NOT NULL,
+    `end` REAL NOT NULL,
+    `taxon_id` INTEGER NOT NULL,
+    `location_id` INTEGER NOT NULL,
+    `date` TEXT NOT NULL,
+    `notes` TEXT,
+    `confidence` REAL NOT NULL,
+    `verified` BOOLEAN DEFAULT 0 CHECK (`verified` IN (0, 1)),
+    FOREIGN KEY(taxon_id) REFERENCES taxon(taxon_id),
+    FOREIGN KEY(location_id) REFERENCES location(location_id)
+);

daemon/database/structure-sqlite.sql

@@ -2,21 +2,21 @@
 
 /** Taxon table */
 CREATE TABLE IF NOT EXISTS taxon (
-    taxon_id INTEGER PRIMARY KEY,
+    taxon_id INTEGER PRIMARY KEY NOT NULL,
     scientific_name TEXT NOT NULL,
     common_name TEXT NOT NULL
 );
 
 /** Location table */
 CREATE TABLE IF NOT EXISTS location (
-    location_id INTEGER PRIMARY KEY,
+    location_id INTEGER PRIMARY KEY NOT NULL,
     latitude REAL NOT NULL,
     longitude REAL NOT NULL
 );
 
 /** Observation table */
 CREATE TABLE IF NOT EXISTS observation (
-    `observation_id` INTEGER PRIMARY KEY,
+    `observation_id` INTEGER PRIMARY KEY NOT NULL,
     `audio_file` TEXT NOT NULL,
     `start` REAL NOT NULL,
     `end` REAL NOT NULL,
@@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS observation (
     `date` TEXT NOT NULL,
     `notes` TEXT,
     `confidence` REAL NOT NULL,
-    `verified` BOOLEAN NOT NULL CHECK (`verified` IN (0, 1)) DEFAULT 0,
+    `verified` BOOLEAN DEFAULT 0 CHECK (`verified` IN (0, 1)),
     FOREIGN KEY(taxon_id) REFERENCES taxon(taxon_id),
     FOREIGN KEY(location_id) REFERENCES location(location_id)
 );

daemon/plotter/chart.py

@@ -1,76 +1,74 @@
 #! /usr/bin/env python3
 
-from curses import def_prog_mode
 import sqlite3
-from xml.sax.handler import feature_external_ges
 import pandas as pd
 import matplotlib.pyplot as plt
 from matplotlib.colors import LogNorm
 import seaborn as sns
 from datetime import datetime
+import os
+import glob
 
 CONFIG = {
     "readings": 10,
     "palette": "Greens",
+    "db": "./var/db.sqlite",
+    "date": datetime.now().strftime("%Y-%m-%d"),
+    "charts_dir": "./var/charts"
 }
 
 db = None
+
 def get_database():
     global db
     if db is None:
-        db = sqlite3.connect('/home/ortion/Desktop/db.sqlite')
+        db = sqlite3.connect(CONFIG["db"])
     return db
     
-
+def chart(date):
-def get_detection_hourly(date):
+    db  = get_database()
-    db = get_database()
+    df = pd.read_sql_query(f"""SELECT common_name, date, location_id, confidence 
-    df = pd.read_sql_query("""SELECT common_name, date, location_id, confidence 
                             FROM observation
                             INNER JOIN taxon 
-                            ON observation.taxon_id = taxon.taxon_id""", db)
+                            ON observation.taxon_id = taxon.taxon_id 
-
+                            WHERE STRFTIME("%Y-%m-%d", `date`) = '{date}'""", db)
     df['date'] = pd.to_datetime(df['date'])
     df['hour'] = df['date'].dt.hour
     df['date'] = df['date'].dt.date
     df['date'] = df['date'].astype(str)
-
     df_on_date = df[df['date'] == date]
-    return df_on_date
     
+    top_on_date = (df_on_date['common_name'].value_counts()[:CONFIG['readings']])
+    if top_on_date.empty:
+        print("No observations on {}".format(date))
+        return
+    else:
+        print(f"Found observations on {date}")
         
-def get_top_species(df, limit=10):
+    df_top_on_date = df_on_date[df_on_date['common_name'].isin(top_on_date.index)]
-    return df['common_name'].value_counts()[:CONFIG['readings']]
 
-
+    # Create a figure with 2 subplots
-def get_top_detections(df, limit=10):
+    fig, axs = plt.subplots(1, 2, figsize=(20, 5), gridspec_kw=dict(
-    df_top_species = get_top_species(df, limit=limit)
+        width_ratios=[2, 6]))
-    return df[df['common_name'].isin(df_top_species.index)]
-
-
-def get_frequence_order(df, limit=10):
-    pd.value_counts(df['common_name']).iloc[:limit]
-
-def presence_chart(date, filename):
-    df_detections = get_detection_hourly(date)
-    df_top_detections = get_top_detections(df_detections, limit=CONFIG['readings'])
-    fig, axs = plt.subplots(1, 2, figsize=(15, 4), gridspec_kw=dict(
-        width_ratios=[3, 6]))
     plt.subplots_adjust(left=None, bottom=None, right=None,
                         top=None, wspace=0, hspace=0)
 
-    frequencies_order = get_frequence_order(df_detections, limit=CONFIG["readings"])
+    # Get species frequencies
+    frequencies_order = pd.value_counts(df_top_on_date['common_name']).iloc[:CONFIG['readings']].index
     # Get min max confidences
-    confidence_minmax = df_detections.groupby('common_name')['confidence'].max()
+    confidence_minmax = df_top_on_date.groupby('common_name')['confidence'].max()
+    confidence_minmax = confidence_minmax.reindex(frequencies_order)
     # Norm values for color palette
     norm = plt.Normalize(confidence_minmax.values.min(),
-                         confidence_minmax.values.max())
+                            confidence_minmax.values.max())
+
     colors = plt.cm.Greens(norm(confidence_minmax))
-    plot = sns.countplot(y='common_name', data=df_top_detections, palette=colors, order=frequencies_order, ax=axs[0])
+    plot = sns.countplot(y='common_name', data=df_top_on_date, palette=colors, order=frequencies_order, ax=axs[0])
 
     plot.set(ylabel=None)
     plot.set(xlabel="Detections")
 
-    heat = pd.crosstab(df_top_detections['common_name'], df_top_detections['hour'])
+    heat = pd.crosstab(df_top_on_date['common_name'], df_top_on_date['hour'])
     # Order heatmap Birds by frequency of occurrance
     heat.index = pd.CategoricalIndex(heat.index, categories=frequencies_order)
     heat.sort_index(level=0, inplace=True)
@@ -94,8 +92,7 @@ def presence_chart(date, filename):
         linewidth=0.5,
         linecolor="Grey",
         ax=axs[1],
-        yticklabels=False
+        yticklabels=False)
-    )
     plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7)
 
     for _, spine in plot.spines.items():
@@ -103,18 +100,25 @@ def presence_chart(date, filename):
 
     plot.set(ylabel=None)
     plot.set(xlabel="Hour of day")
-    fig.subplots_adjust(top=0.9)
+    plt.suptitle(f"Top {CONFIG['readings']} species on {date}", fontsize=14)
-    plt.suptitle(f"Top {CONFIG['readings']} species (Updated on {datetime.now().strftime('%Y/%m-%d %H:%M')})")
+    plt.text(15, 11, f"(Updated on {datetime.now().strftime('%Y/%m-%d %H:%M')})")
-
+    plt.savefig(f"./var/charts/chart_{date}.png", dpi=300)
-    plt.savefig(filename)
+    print(f"Plot for {date} saved.")
     plt.close()
 
 def main():
-    date = datetime.now().strftime('%Y%m%d')
+    done_charts = glob.glob(f"{CONFIG['charts_dir']}/*.png")
-    presence_chart(date, f'./var/charts/chart_{date}.png')
+    last_modified = max(done_charts, key=os.path.getctime)
-    # print(get_top_detections(get_detection_hourly(date), limit=10))
+    last_modified_date = last_modified.split("_")[-1].split(".")[0]
-    if not db is None:
+    missing_dates = pd.date_range(start=last_modified_date, end=CONFIG['date'], freq='D')
+    print(missing_dates)
+    for missing_date in missing_dates:
+        date = missing_date.strftime("%Y-%m-%d")
+        chart(date)
+    chart(CONFIG['date'])
+    if db is not None:
         db.close()
+    print("Done.")
 
 if __name__ == "__main__":
     main()

daemon/systemd/templates/birdnet_miner.service

@@ -1,13 +0,0 @@
-[Unit]
-Description=BirdNET-stream miner service
-
-[Service]
-Type=simple
-User=<USER>
-Group=<GROUP>
-WorkingDirectory=<DIR>
-ExecStart=bash ./daemon/birdnet_miner.sh
-RemainAfterExit=yes
-
-[Install]
-WantedBy=multi-user.target

daemon/systemd/templates/birdnet_miner.timer

@@ -1,9 +0,0 @@
-[Unit]
-Description=BirdNET-stream miner Timer
-
-[Timer]
-OnCalendar=*:0/15
-Unit=birdnet_miner.service
-
-[Install]
-WantedBy=basic.target

daemon/systemd/templates/birdnet_plotter.service

@@ -5,7 +5,7 @@ Description=BirdNET-stream plotter
 User=<USER>
 Group=<GROUP>
 WorkingDirectory=<DIR>
-ExecStart=./.venv/birdnet-stream/bin/python3 ./daemon/plotter/chart.py
+ExecStart=<VENV>/bin/python3 ./daemon/plotter/chart.py
 Type=simple
 
 [Install]

daemon/systemd/templates/birdnet_ttyd.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=BirdNET-stream logs
+After=syslog.target
+After=network.target
+
+[Service]
+User=<USER>
+Group=<GROUP>
+ExecStart=/opt/ttyd -p 7681 -c birdnet:secret  -t disableReconnect=true --readonly journalctl -feu birdnet_\*
+Restart=always
+Type=simple
+
+[Install]
+WantedBy=multi-user.target

docker-compose.yml

@@ -1,36 +1,119 @@
-version: '3.8'
+version: '3.9'
-
-networks:
-  birdnet_network:
 
 services:
-  # database:
+  # recording: 
-  #   container_name: birdnet_database
+  #   container_name: birdnet_recording
-  #   image:
+  #   build:
+  #     context: .
+  #     dockerfile: ./docker/recording/Dockerfile
+  #   restart: unless-stopped
+  #   environment:
+  #     - CHUNK_FOLDER=${CHUNK_FOLDER:-/media/birdnet/records}
+  #   volumes:
+  #     - ${RECORDS_DIR:-/media/birdnet/records}:${RECORS_FOLDER:-/media/birdnet/records}
+  #   # Allow container to access to the hosts microphone
+  #   devices:
+  #     - /dev/snd:/dev/snd
+  # analyzer:
+  #   container_name: birdnet_analyzer
+  #   build:
+  #     context: ./analyzer/
+  #     dockerfile: ./Dockerfile
 
-  php:
+  db:
-    container_name: birdnet_php
+    container_name: birdnet_database
-    image: php:8.1-fpm
+    image: mariadb:latest
+    command: mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
     ports:
-      - "${PHP_FPM_PORT:-9001}:9000"
+      - ${DATABASE_PORT:-3306}:3306
+    networks:
+      - birdnet_network
+    environment:
+      MYSQL_ROOT_PASSWORD: ${DATABASE_ROOT_PASSWORD:-secret}'
+      MYSQL_USER: ${DATABASE_USER:-birdnet}
+      MYSQL_PASSWORD: ${DATABASE_PASSWORD:-secret}
+    volumes:
+      - ./docker/database/init:/docker-entrypoint-initdb.d
+    restart: unless-stopped
 
+  php-fpm:
+    container_name: birdnet_php-fpm
+    build:
+      context: .
+      dockerfile: ./docker/php-fpm/Dockerfile
+    ports:
+      - '${PHP_FPM_PORT:-9000}:9000'
+    networks:
+      - birdnet_network
+    environment:
+      - APP_ENV=${APP_ENV:-prod}
+      - APP_DEBUG=${APP_DEBUG:-true}
+      - DATABASE_DEFAULT_URL=mysql://${DATABASE_USER:-birdnet}:${DATABASE_PASSWORD:-secret}@${DATABASE_HOST:-birdnet_database}:${DATABASE_PORT:-3306}/birdnet_default
+      - DATABASE_OBSERVATIONS_URL=mysql://${DATABASE_USER:-birdnet}:${DATABASE_PASSWORD:-secret}@${DATABASE_HOST:-birdnet_database}:${DATABASE_PORT:-3306}/birdnet_observations
+    restart: unless-stopped
+    volumes:
+      - birdnet_app:${PROJECT_ROOT:-/opt/birdnet}
+
+  symfony:
+    container_name: birdnet_symfony
+    networks:
+      - birdnet_network
+    build:
+      context: .
+      dockerfile: ./docker/symfony/Dockerfile
+      args:
+        - DATABASE_DEFAULT_URL=mysql://${DATABASE_USER:-birdnet}:${DATABASE_PASSWORD:-secret}@${DATABASE_HOST:-birdnet_database}:${DATABASE_PORT:-3306}/birdnet_default
+        - DATABASE_OBSERVATIONS_URL=mysql://${DATABASE_USER:-birdnet}:${DATABASE_PASSWORD:-secret}@${DATABASE_HOST:-birdnet_database}:${DATABASE_PORT:-3306}/birdnet_observations
+        - RECORDS_DIR=/media/birdnet/records
+        - CHARTS_DIR=/media/birdnet/charts
+    restart: on-failure
+    volumes:
+      - birdnet_app:${PROJECT_ROOT:-/opt/birdnet}
+      - birdnet_records:${RECORDS_DIR:-/media/birdnet/records}
+    depends_on:
+      - db
 
   nginx:
     container_name: birdnet_nginx
+    hostname: ${SERVER_NAME:-birdnet.local}
     build:
-      context: ./docker/
+      context: .
-    environment:
+      dockerfile: ./docker/nginx/Dockerfile
-      SERVER_NAME: ${SERVER_NAME:-birdnet.local}
+      args:
-      PHP_FPM_PORT: ${PHP_FPM_PORT:-9001}
+        - SERVER_NAME=${SERVER_NAME:-birnet.local}
-    restart: unless-stopped
+        - SYMFONY_PUBLIC=/opt/birdnet/www/public
-    volumes:
+        - CHARTS_DIR=/media/birdnet/charts
-      - ./www:/var/www/birdnet/
+        - RECORDS_DIR=/media/birdnet/records
-      - ./www/nginx.conf:/etc/nginx/conf.d/birdnet.conf
+        - PHP_FPM_HOST=birdnet_php-fpm
+        - PHP_FPM_PORT=9000
     ports:
-      - "81:80"
+      - ${HTTP_PORT:-80}:80
-    dependends_on:
+      - ${HTTPS_PORT:-443}:443
-      - php
+    volumes:
+      - birdnet_app:/opt/birdnet
+      - birdnet_records:/media/data/records
+    networks:
+      birdnet_network:
+        ipv4_address: ${IP_ADDRESS:-172.25.0.101}
+        aliases:
+          - ${SERVER_NAME:-birdnet.local}
+    restart: unless-stopped
+    depends_on:
+      - symfony
+      - php-fpm
 
-  birdnet:
+networks:
-    container_name: birdnet_analyzer
+  birdnet_network:
-    image: 
+    driver: bridge
+    ipam:
+      config:
+        - subnet: ${IP_SUBNET:-172.25.0.0/24}
+
+volumes:
+  birdnet_app:
+
+  birdnet_records:
+    driver_opts:
+      type: none
+      device: ${RECORDS_DIR:-/media/data/records}
+      o: bind

docker/all/Dockerfile

@@ -4,17 +4,24 @@ FROM debian:bullseye
 ENV REPOSITORY=${REPOSITORY:-https://github.com/UncleSamulus/BirdNET-stream.git}
 # DEBUG defaults to 1 for descriptive DEBUG logs, 0 for error logs only 
 ENV DEBUG=${DEBUG:-1} 
-RUN useradd birdnet
 WORKDIR /home/birdnet
+RUN useradd -m -s /bin/bash -G sudo birdnet
+USER birdnet
 
 # Upgrade system
 RUN apt-get update && apt-get upgrade -y
 
-# Install curl
+# Install some dependencies
 RUN apt-get install -y \
+    sudo \
+    git \
     curl \
-    sudo
+    bash \
+    vim \
+    systemctl
 
-RUN curl -sL https://raw.githubusercontent.com/UncleSamulus/BirdNET-stream/master/install.sh | bash
+COPY ./install.sh install.sh
 
-USER birdnet
+RUN ./install.sh
+
+EXPOSE 443

docker/all/README.md

@@ -9,7 +9,7 @@
 ```bash
 git clone https://github.com/UncleSamulus/BirdNET-stream.git
 cd ./BirdNET-stream/docker/all
-docker build -t "birdnet_all:latest" .
+docker build -t "birdnet_all:latest" -f ./docker/all/Dockerfile .
 ```
 
 If `docker` command does not work because of unsufficient permissions, you could add your user to `docker` group:

docker/database/init/00-init-databases.sql

@@ -0,0 +1,5 @@
+CREATE DATABASE IF NOT EXISTS birdnet_default;
+CREATE DATABASE IF NOT EXISTS birdnet_observations;
+
+GRANT ALL ON `birdnet_observations`.* TO 'birdnet'@'%' IDENTIFIED BY 'secret';
+GRANT ALL ON `birdnet_default`.* TO 'birdnet'@'%' IDENTIFIED BY 'secret';

docker/nginx/Dockerfile

@@ -0,0 +1,31 @@
+FROM nginx
+
+ARG SERVER_NAME
+ARG PROJECT_ROOT
+ARG SYMFONY_PUBLIC
+ARG CHARTS_DIR
+ARG RECORDS_DIR
+ARG PHP_FPM_HOST
+ARG PHP_FPM_PORT
+
+RUN apt-get update && apt-get upgrade -y \
+    && apt-get install -y nginx-full
+
+USER root
+COPY "docker/nginx/nginx.conf.template" "/etc/nginx/sites-available/birdnet.conf" 
+RUN ln -s /etc/nginx/sites-available/birdnet.conf /etc/nginx/sites-enabled/birdnet.conf
+RUN sed -i "s|<SERVER_NAME>|${SERVER_NAME}|g" /etc/nginx/sites-available/birdnet.conf \
+    && sed -i "s|<PHP_FPM_HOST>|${PHP_FPM_HOST}|g" /etc/nginx/sites-available/birdnet.conf \
+    && sed -i "s|<PHP_FPM_PORT>|${PHP_FPM_PORT}|g" /etc/nginx/sites-available/birdnet.conf \
+    && sed -i "s|<SYMFONY_PUBLIC>|${SYMFONY_PUBLIC}|g" /etc/nginx/sites-available/birdnet.conf \
+    && sed -i "s|<RECORDS_DIR>|${RECORDS_DIR}|g" /etc/nginx/sites-available/birdnet.conf \
+    && sed -i "s|<CHARTS_DIR>|${CHARTS_DIR}|g" /etc/nginx/sites-available/birdnet.conf
+RUN mkdir -p /etc/nginx/certs/birdnet 
+WORKDIR /etc/nginx/certs/birdnet
+RUN openssl req -x509 -newkey rsa:4096 -keyout privkey.pem -out fullchain.pem -sha256 -days 365 -nodes --subj "/CN=${SERVER_NAME}"
+RUN sed -i "s|<CERTIFICATE>|/etc/nginx/certs/birdnet/fullchain.pem|g" /etc/nginx/sites-available/birdnet.conf \
+    && sed -i "s|<PRIVATE_KEY>|/etc/nginx/certs/birdnet/privkey.pem|g" /etc/nginx/sites-available/birdnet.conf
+
+EXPOSE 443
+EXPOSE 80
+CMD ["nginx", "-g", "daemon off;"]

docker/nginx/nginx.conf.template

@@ -0,0 +1,52 @@
+server {
+	listen 80;
+	server_name <SERVER_NAME>;
+
+	location / {
+		return 302 https://$host$request_uri;
+	}
+
+    location /.well-known/acme-challenge {
+        alias /var/www/html/.well-known/acme-challenge;
+        allow all;
+    }
+}
+
+server {
+	listen 443 ssl;
+	server_name <SERVER_NAME>;
+
+	fastcgi_buffers  16 16k;
+    fastcgi_buffer_size  32k;	
+
+	root  <SYMFONY_PUBLIC>;
+
+	ssl_certificate  <CERTIFICATE>;
+	ssl_certificate_key  <PRIVATE_KEY>;
+
+	index index.php;
+
+	location / {
+    	try_files $uri $uri/ /index.php$is_args$args;
+    }
+
+	location ^~ /media/records {
+		autoindex on;
+		alias <RECORDS_DIR>;
+	}
+	
+	location ^~ /media/charts {
+		autoindex on;
+		alias <CHARTS_DIR>;
+	}
+
+    location ~ \.php$ {
+		fastcgi_split_path_info ^(.+\.php)(/.+)$;
+		fastcgi_pass <PHP_FPM_HOST>:<PHP_FPM_PORT>;
+		fastcgi_index index.php;
+		include fastcgi.conf;
+    }
+
+	access_log /var/log/nginx/birdnet-access.log;
+	error_log  /var/log/nginx/birdnet-error.log error;
+}

docker/php-fpm/Dockerfile

@@ -0,0 +1,8 @@
+ARG PHP_VERSION=${PHP_VERSION:-8.1}
+
+FROM php:${PHP_VERSION}-fpm
+
+RUN apt-get update && apt-get upgrade -y 
+RUN docker-php-ext-install pdo pdo_mysql
+
+EXPOSE 9000

docker/recording/Dockerfile

@@ -1,16 +1,25 @@
 # Recording container for BirdNET-stream
-# Reference: https://leimao.github.io/blog/Docker-Container-Audio/
+# References:
+# - https://leimao.github.io/blog/Docker-Container-Audio/
+# - https://askubuntu.com/questions/972510/how-to-set-alsa-default-device-to-pulseaudio-sound-server-on-docker
 
 FROM debian:bullseye
 
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install packages dependencies
-
+RUN apt-get update && apt-get upgrade -y \
-RUN apt-get update && \
+    && apt-get install -y \
-    apt-get install apt-utils \
+    --no-install-recommends \
-    && apt-get install -y --no-install-recommends \
     libasound2 \
     alsa-utils \
     libsndfile1-dev \
+    && apt-get install -y ffmpeg \
     && apt-get clean
+
+RUN mkdir -p /opt/birdnet/
+WORKDIR /opt/birdnet/
+COPY config ./config
+COPY daemon/birdnet_recording.sh /usr/local/bin/birdnet_recording.sh
+
+ENTRYPOINT ["/usr/local/bin/birdnet_recording.sh"]

docker/symfony/Dockerfile

@@ -0,0 +1,75 @@
+ARG PHP_VERSION=${PHP_VERSION:-8.1}
+
+FROM php:${PHP_VERSION}
+
+ARG PROJECT_ROOT
+ARG NODE_VERSION
+ARG RECORDS_DIR
+ARG CHARTS_DIR
+ARG DATABASE_DEFAULT_URL
+ARG DATABASE_OBSERVATIONS_URL
+
+ENV PHP_VERSION=${PHP_VERSION:-8.1} \
+    NODE_VERSION=${NODE_VERSION:-16.17.0} \
+    PROJECT_ROOT=${PROJECT_ROOT:-/opt/birdnet} \
+    RECORDS_DIR=${RECORDS_DIR:-/media/data/birdnet/records} \
+    CHARTS_DIR=${CHARTS_DIR:-/media/data/birdnet/charts} \
+    DATABASE_DEFAULT_URL=${DATABASE_DEFAULT_URL:-mysql://birdnet:secret@birdnet_database/birdnet} \
+    DATABASE_OBSERVATIONS_URL=${DATABASE_OBSERVATIONS_URL:-mysql://birdnet:secret@birdnet_database/birdnet_observations} 
+
+ENV APP_ENV=${APP_ENV:-prod}
+ENV APP_DEBUG=${APP_DEBUG:-0}
+
+# RUN apt-get update && apt-get upgrade -y \
+#     && apt-get install -y \
+#     curl \
+#     zip \
+#     unzip \
+#     zlib1g-dev \
+#     libzip-dev \
+#     git \
+#     vim \
+#     && apt-get clean 
+
+# RUN docker-php-ext-install zip pdo_mysql
+
+# # Install composer
+# RUN php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" \
+#     && php composer-setup.php --install-dir=/usr/local/bin --filename=composer
+
+# # Install nodejs and npm
+# ENV NVM_DIR="/usr/local/nvm"
+# RUN mkdir ${NVM_DIR}
+# RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash
+# RUN . "$NVM_DIR/nvm.sh" \
+#     && nvm install ${NODE_VERSION} \
+#     && nvm use ${NODE_VERSION} \
+#     && nvm alias default ${NODE_VERSION} \
+#     && npm install -g yarn
+# ENV PATH="$PATH:/usr/local/nvm/versions/node/v${NODE_VERSION}/bin"
+
+# Change permissions for the home folder of www-data (used for composer cache)
+RUN chown -R www-data:www-data /var/www
+COPY . ${PROJECT_ROOT}
+WORKDIR ${PROJECT_ROOT}/www
+RUN rm -rf {vendor,node_modules}
+RUN chown -R www-data:www-data .
+
+USER www-data
+# Setup .env
+RUN cp .env.local.example .env.local
+RUN sed -i "s/^APP_ENV=.*/APP_ENV=prod/g" .env.local \
+&& sed -i "s/^APP_DEBUG=.*/APP_DEBUG=0/g" .env.local \
+&& sed -i "s/^APP_SECRET=.*/APP_SECRET=${APP_SECRET}/g" .env.local \
+&& sed -i "s|^DATABASE_DEFAULT_URL=.*|DATABASE_DEFAULT_URL=${DATABASE_DEFAULT_URL}|g" .env.local \
+&& sed -i "s|^DATABASE_OBSERVATIONS_URL=.*|DATABASE_OBSERVATIONS_URL=${DATABASE_OBSERVATIONS_URL}|g" .env.local \
+&& sed -i "s|^RECORDS_DIR=.*|RECORDS_DIR=${RECORDS_DIR}|g" .env.local \
+&& sed -i "s|^CHARTS_DIR=.*|CHARTS_DIR=${CHARTS_DIR}|g" .env.local
+
+# # # Install yarn dependencies
+# RUN . "$NVM_DIR/nvm.sh" && yarn install && yarn build
+
+# # Install composer dependencies
+# RUN composer install --no-interaction --prefer-dist --optimize-autoloader
+# RUN composer dump-env prod
+# RUN composer dump-autoload

docs/DATABASE.md

@@ -0,0 +1,58 @@
+# Setting up the database
+
+There is two database managment systems available: sqlite or mariadb (mysql).
+
+## sqlite
+
+To use sqlite, simply install the sqlite3 package, if it is not already installed on the machine that runs BirdNET-stream.
+
+```bash
+sudo apt-get install sqlite3
+```
+
+Then fill `config/birdnet.conf` with the proper DATABASE value (you may use any database location):
+
+```bash
+DATABASE="./var/db.sqlite"
+```
+
+## mariadb
+
+To use mariadb, you need to install the mariadb-server package.
+
+```bash
+sudo apt-get install mariadb-server
+```
+
+Then, populate the `config/birdnet.conf` file with the proper DATABASE uri:
+
+```bash
+DATABASE="mysql://user:password@localhost/birdnet_observations"
+```
+
+## Symfony configuration
+
+For both method you need to adapt the file `www/.env.local` to suit your new configuration.
+
+```bash
+cd www
+# If .env.local does not exists:
+cp .env.local.example .env.local
+```
+
+```text
+# .env.local
+# for sqlite (example)
+DATABASE_DEFAULT_URL=sqlite:///%kernel.project_dir%/./var/db-default.sqlite
+DATABASE_OBSERVATIONS_URL=sqlite:///%kernel.project_dir%/../var/db.sqlite
+# for mariadb (example)
+DATABASE_DEFAULT_URL=mysql://user:password@localhost/birdnet_default
+DATABASE_OBSERVATIONS_URL=mysql://user:password@localhost/birdnet_observations
+```
+
+## PHP modules
+
+For symfony to work, make sure you have the required modules according to each method:
+
+- pdo_sqlite
+- pdo_mysql

docs/DOCKER.md

@@ -0,0 +1,54 @@
+# Use docker to run BirdNET-stream
+
+There are two ways to run BirdNET-stream using docker: a "all in one" container, running all services on the same container, or using a splitted approach, running each service on a separate container.
+
+## Prerequisites
+
+- docker
+- docker-compose (for splitted approach)
+- git
+
+## Using the all in one container (not working yet)
+
+The all in one container is a container that runs all services on the same container.
+
+You can follow the instructions in [./docker/all/README.md](./docker/all/README.md) to create this container.
+
+## Using the splitted approach (recommended)
+
+The splitted approach uses docker-compose and a docker container for each service.
+
+This is the recommended approach to run BirdNET-stream while using docker.
+
+First of of all, you need to clone the repository.
+
+```bash
+mkdir ~/Documents/BirdNET-stream
+cd ~/Documents/BirdNET-stream
+git clone -b main https://github.com/UncleSamulus/BirdNET-stream.git .
+```
+
+Then, create your dotenv file and populate it with your own configuration (for instance, generate random passwords and add them to .env credentials):
+
+```bash
+cp .env.example .env
+```
+
+You may need to adapt the listening ports of the services or other configuration parameters. 
+In general all variables stated with ${VARIABLE:-default} inside [../docker-compose.yml](../docker-compose.yml) can be override in the .env file using `VARIABLE=value`.
+
+Once that is done, you can build and start docker services:
+
+```bash
+# Build images (first time only)
+docker compose build
+# Run
+docker compose up # add `-d`, to run in background
+# Stop
+docker compose down
+```
+
+For a one liner:
+```bash
+docker compose up --build
+```

install.sh

@@ -5,13 +5,20 @@ set -e
 
 DEBUG=${DEBUG:-0}
 
-REQUIREMENTS="git ffmpeg python3-pip python3-dev"
+REQUIREMENTS="git wget ffmpeg python3 python3-pip python3-dev python3-venv zip unzip sqlite3"
 REPOSITORY=${REPOSITORY:-https://github.com/UncleSamulus/BirdNET-stream.git}
+BRANCH=${BRANCH:-main}
+WORKDIR="$(pwd)/BirdNET-stream"
+PYTHON_VENV="./.venv/birdnet-stream"
 
 debug() {
-    if [ $DEBUG -eq 1 ]; then
+    [[ $DEBUG -eq 1 ]] && echo "$@"
-        echo "$1"
+}
-    fi
+
+add_birdnet_user() {
+    sudo useradd -m -s /bin/bash -G sudo birdnet
+    sudo usermod -aG birdnet $USER
+    sudo usermod -aG birdnet www-data
 }
 
 install_requirements() {
@@ -19,11 +26,11 @@ install_requirements() {
     # Install requirements
     missing_requirements=""
     for requirement in $requirements; do
-        if ! dpkg -s $requirement >/dev/null 2>&1; then
+        if ! dpkg -s "$requirement" >/dev/null 2>&1; then
             missing_requirements="$missing_requirements $requirement"
         fi
     done
-    if [ -n "$missing_requirements" ]; then
+    if [[ -n "$missing_requirements" ]]; then
         debug "Installing missing requirements: $missing_requirements"
         sudo apt-get install -y $missing_requirements
     fi
@@ -32,73 +39,85 @@ install_requirements() {
 # Install BirdNET-stream
 install_birdnetstream() {
     # Check if repo is not already installed
-    workdir=$(pwd)
+    if [[ -d "$WORKDIR" ]]; then
-    if [ -d "$workdir/BirdNET-stream" ]; then
+        debug "BirdNET-stream is already installed, use update script (not implemented yet)"
-        debug "BirdNET-stream is already installed"
     else
+        debug "Installing BirdNET-stream"
+        debug "Creating BirdNET-stream directory"
+        mkdir -p "$WORKDIR"
         # Clone BirdNET-stream
+        cd "$WORKDIR"
         debug "Cloning BirdNET-stream from $REPOSITORY"
-        git clone --recurse-submodules $REPOSITORY
+        git clone -b "$BRANCH" --recurse-submodules "$REPOSITORY" .
-        # Install BirdNET-stream
+        debug "Creating python3 virtual environment $PYTHON_VENV"
+        python3 -m venv $PYTHON_VENV
+        debug "Activating $PYTHON_VENV"
+        source "$PYTHON_VENV/bin/activate"
+        debug "Installing python packages"
+        pip3 install -U pip
+        pip3 install -r requirements.txt
+        debug "Creating ./var directory"
+        mkdir -p ./var/{charts,chunks/{in,out}}
     fi
-    cd BirdNET-stream
-    debug "Creating python3 virtual environment '$PYTHON_VENV'"
-    python3 -m venv $PYTHON_VENV
-    debug "Activating $PYTHON_VENV"
-    source .venv/birdnet-stream/bin/activate
-    debug "Installing python packages"
-    pip install -U pip
-    pip install -r requirements.txt
 }
 
 # Install systemd services
 install_birdnetstream_services() {
-    cd BirdNET-stream
+    GROUP=birdnet
-    DIR=$(pwd)
+    DIR="$WORKDIR"
-    GROUP=$USER
+    cd "$WORKDIR"
     debug "Setting up BirdNET stream systemd services"
-    services="birdnet_recording.service birdnet_analyzis.service birdnet_miner.timer birdnet_miner.service birdnet_plotter.service birdnet_plotter.timer"
+    services="birdnet_recording.service birdnet_analyzis.service birdnet_plotter.service birdnet_plotter.timer"
     read -r -a services_array <<<"$services"
-
     for service in ${services_array[@]}; do
-        sudo cp daemon/systemd/templates/$service /etc/systemd/system/
+        sudo cp "daemon/systemd/templates/$service" "/etc/systemd/system/"
         variables="DIR USER GROUP"
         for variable in $variables; do
-            sudo sed -i "s|<$variable>|${!variable}|g" /etc/systemd/system/$service
+            sudo sed -i "s|<$variable>|${!variable}|g" "/etc/systemd/system/$service"
         done
     done
+    sudo sed -i "s|<VENV>|$WORKDIR/$PYTHON_VENV|g" "/etc/systemd/system/birdnet_plotter.service"
     sudo systemctl daemon-reload
-    sudo systemctl enable --now birdnet_recording.service birdnet_analyzis.service birdnet_miner.timer birdnet_plotter.timer
+    enabled_services="birdnet_recording.service birdnet_analyzis.service birdnet_plotter.timer"
+    read -r -a services_array <<<"$services"
+    for service in ${services_array[@]}; do
+        debug "Enabling $service"
+        sudo systemctl enable "$service"
+        sudo systemctl start "$service"
+    done
 }
 
 install_php8() {
     # Remove previously installed php version
-    sudo apt-get remove --purge php*
+    sudo apt-get remove --purge php* -y
     # Install required packages for php
     sudo apt-get install -y lsb-release ca-certificates apt-transport-https software-properties-common gnupg2
     # Get php package from sury repo
     echo "deb https://packages.sury.org/php/ $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/sury-php.list
     sudo wget -qO - https://packages.sury.org/php/apt.gpg | sudo gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/debian-php-8.gpg --import
     sudo chmod 644 /etc/apt/trusted.gpg.d/debian-php-8.gpg
-    update
+    sudo apt-get update && sudo apt-get upgrade -y
-    sudo apt-get install php8.1
+    sudo apt-get install -y php8.1
     # Install and enable php-fpm
-    sudo apt-get install php8.1-fpm
+    sudo apt-get install -y php8.1-fpm
-    sudo systemctl enable php8.1-fpm
+    sudo systemctl enable --now php8.1-fpm
     # Install php packages
-    sudo apt-get install php8.1-{sqlite3,curl,intl}
+    sudo apt-get install -y php8.1-{sqlite3,curl,intl,xml,zip}
 }
 
 install_composer() {
-    php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"\nphp -r "if (hash_file('sha384', 'composer-setup.php') === '55ce33d7678c5a611085589f1f3ddf8b3c52d662cd01d4ba75c0ee0459970c2200a51f492d557530c71c15d8dba01eae') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"\nphp composer-setup.php\nphp -r "unlink('composer-setup.php');"
+    php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
-    sudo mv /composer.phar /usr/local/bin/composer
+    php -r "if (hash_file('sha384', 'composer-setup.php') === '55ce33d7678c5a611085589f1f3ddf8b3c52d662cd01d4ba75c0ee0459970c2200a51f492d557530c71c15d8dba01eae') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"
+    php composer-setup.php
+    php -r "unlink('composer-setup.php');"
+    sudo mv composer.phar /usr/local/bin/composer
 }
 
 install_nodejs() {
     # Install nodejs
     curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash
-    export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")"
+    export NVM_DIR="$([[ -z "${XDG_CONFIG_HOME-}" ]] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")"
-    [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm
+    [[ -s "$NVM_DIR/nvm.sh" ]] && \. "$NVM_DIR/nvm.sh" # This loads nvm
     nvm install 16
     nvm use 16
     install_requirements "npm"
@@ -108,7 +127,6 @@ install_nodejs() {
 
 install_web_interface() {
     debug "Setting up web interface"
-    install_requirements "nginx"
     # Install php 8.1
     install_php8
     # Install composer
@@ -116,30 +134,95 @@ install_web_interface() {
     # Install nodejs 16
     install_nodejs
     # Install Symfony web app
-    cd BirdNET-stream
+    cd "$WORKDIR"
     cd www
-    debug "Creating nginx configuration"
-    cp nginx.conf /etc/nginx/sites-available/birdnet-stream.conf
-    sudo mkdir /var/log/nginx/birdnet/
-    echo "Info: Please edit /etc/nginx/sites-available/birdnet-stream.conf to set the correct server name and paths"
-    sudo ln -s /etc/nginx/sites-available/birdnet-stream.conf /etc/nginx/sites-enabled/birdnet-stream.conf
-    sudo systemctl enable --now nginx
-    sudo systemctl restart nginx
     debug "Retrieving composer dependencies"
     composer install
+    debug "PHP dependencies installed"
     debug "Installing nodejs dependencies"
     yarn install
+    debug "npm dependencies installed"
     debug "Building assets"
     yarn build
+    debug "Webpack assets built"
     debug "Web interface is available"
     debug "Please restart nginx after double check of /etc/nginx/sites-available/birdnet-stream.conf"
 }
 
+setup_http_server() {
+    debug "Setting up HTTP server"
+    install_requirements "nginx"
+    debug "Setup nginx server"
+    cd "$WORKDIR"
+    cd www
+    debug "Creating nginx configuration"
+    sudo cp nginx.conf.template /etc/nginx/sites-available/birdnet-stream.conf
+    sudo mkdir -p /var/log/nginx/birdnet/
+    if [[ -f "/etc/nginx/sites-enabled/birdnet-stream.conf" ]]; then
+        sudo unlink /etc/nginx/sites-enabled/birdnet-stream.conf
+    fi
+    debug "Enable birdnet.lan domain"
+    sudo ln -s /etc/nginx/sites-available/birdnet-stream.conf /etc/nginx/sites-enabled/birdnet-stream.conf
+    debug "INFO: Please edit /etc/nginx/sites-available/birdnet-stream.conf to set the correct server name and paths"
+    debug "Setup nginx variables the best way possible"
+    sudo sed -i "s|<SYMFONY_PUBLIC>|$WORKDIR/www/public/|g" /etc/nginx/sites-available/birdnet-stream.conf
+    sudo sed -i "s|<RECORDS_DIR>|$CHUNK_FOLDER/out|g" /etc/nginx/sites-available/birdnet-stream.conf
+    sudo sed -i "s|<CHARTS_DIR>|$WORKDIR/var/charts|g" /etc/nginx/sites-available/birdnet-stream.conf
+    debug "Generate self signed certificate"
+    CERTS_LOCATION="/etc/nginx/certs/birdnet"
+    sudo mkdir -p "$CERTS_LOCATION"
+    cd $CERTS_LOCATION
+    sudo openssl req -x509 -newkey rsa:4096 -keyout privkey.pem -out fullchain.pem -sha256 -days 365 -nodes --subj '/CN=birdnet.lan'
+    sudo sed -i "s|<CERTIFICATE>|$CERTS_LOCATION/fullchain.pem|g" /etc/nginx/sites-available/birdnet-stream.conf
+    sudo sed -i "s|<PRIVATE_KEY>|$CERTS_LOCATION/privkey.pem|g" /etc/nginx/sites-available/birdnet-stream.conf
+    sudo systemctl enable --now nginx
+    sudo systemctl restart nginx
+    cd -
+}
+
+change_value() {
+    local variable_name
+    variable_name="$1"
+    local variable_new_value
+    variable_new_value="$2"
+    local variable_filepath="$3"
+    sed -i "s|$variable_name=.*|$variable_name=\"$variable_new_value\"|g" "$variable_filepath"
+}
+
+install_config() {
+    debug "Updating config"
+    cd "$WORKDIR"
+    cp ./config/birdnet.conf.example ./config/birdnet.conf
+    config_filepath="$WORKDIR/config/birdnet.conf"
+    change_value "DIR" "$WORKDIR" "$config_filepath"
+    change_value "PYTHON_VENV" "$PYTHON_VENV" "$config_filepath"
+    change_value "AUDIO_RECORDING" "true" "$config_filepath"
+    source "$config_filepath"
+    cd www
+    debug "Setup webapp .env"
+    cp .env.local.example .env.local
+    change_value "RECORDS_DIR" "$CHUNKS_FOLDER" ".env.local"
+}
+
+update_permissions() {
+    debug "Updating permissions (may not work properly)"
+    cd $WORKDIR
+    sudo chown -R $USER:birdnet "$WORKDIR"
+    sudo chown -R $USER:birdnet "$CHUNK_FOLDER"
+    sudo chmod -R 755 "$CHUNK_FOLDER"
+}
+
 main() {
-    install_requirements $REQUIREMENTS
+    install_requirements "$REQUIREMENTS"
     install_birdnetstream
     install_birdnetstream_services
     install_web_interface
+    setup_http_server
+    install_config
+    debug "Run loginctl enable-linger for $USER"
+    loginctl enable-linger
+    update_permissions
+    debug "Installation done"
 }
 
 main

uninstall.sh

@@ -0,0 +1,49 @@
+#! /usr/bin/env bash
+# Standard uninstallation script for BirdNET-stream installed on Debian Based Linux distros
+set -e
+# set -x 
+DEBUG=${DEBUG:-0}
+debug() {
+    [[ $DEBUG -eq 1 ]] && echo "$@"
+}
+
+if [[ -f ./config/birdnet.conf ]]; then
+    source ./config/birdnet.conf
+fi
+
+WORKDIR=${WORKDIR:-$(pwd)/BirdNET-stream}
+
+# Remove systemd services
+uninstall_birdnet_services() {
+    debug "Removing systemd services"
+    services=(birdnet_recording.service
+            birdnet_streaming.service
+            birdnet_miner.service
+            birdnet_miner.timer
+            birdnet_plotter.service
+            birdnet_plotter.timer)
+    for service in "$services"; do
+        debug "Stopping $service"
+        sudo systemctl stop "$service"
+        sudo systemctl disable "$service"
+        sudo rm -f "/etc/systemd/system/$service"
+        sudo systemctl daemon-reload
+    done
+    debug "Done removing systemd services"
+}
+
+uninstall_webapp() {
+    debug "Removing webapp"
+    debug "Removing nginx server configuration"
+    sudo unlink /etc/nginx/sites-enabled/birdnet-stream.conf
+    sudo systemctl restart nginx
+}
+
+main() {
+    echo "WARNING: This will remove all BirdNET-stream related files and services. \
+    Note that it may forget some special configuration."
+    uninstall_webapp
+    uninstall_birdnet_services
+}
+
+main

utils/fix_permissions.sh

@@ -1,15 +1,13 @@
 #! /usr/bin/env bash
-
+# Fix permissions on BirdNET-stream files when messed up
 set -e
 
-DEBUG=${DEBUG:-1}
+DEBUG=${DEBUG:-0}
 debug() {
-    if [ $DEBUG -eq 1 ]; then
+    [ $DEBUG -eq 1 ] && echo "$@"
-        echo "$1"
-    fi
 }
 
-config_filepath="./config/analyzer.conf"
+config_filepath="./config/birdnet.conf"
 
 if [ -f "$config_filepath" ]; then
     source "$config_filepath"
@@ -18,7 +16,6 @@ else
     exit 1
 fi
 
-
 GROUP=birdnet
 
 sudo chown -R $USER:$GROUP $CHUNK_FOLDER

www/.env

@@ -25,11 +25,14 @@ APP_SECRET=8bd3643031a08d0cd34e6fd3f680fb22
 #
 # DATABASE_URL="sqlite:///%kernel.project_dir%/var/data.db"
 # DATABASE_URL="mysql://app:!ChangeMe!@127.0.0.1:3306/app?serverVersion=8&charset=utf8mb4"
-DATABASE_URL="postgresql://app:!ChangeMe!@127.0.0.1:5432/app?serverVersion=14&charset=utf8"
+# DATABASE_URL="postgresql://app:!ChangeMe!@127.0.0.1:5432/app?serverVersion=14&charset=utf8"
+DATABASE_DEFAULT_URL=sqlite:///%kernel.project_dir%/./var/db-default.sqlite
+DATABASE_OBSERVATIONS_URL=sqlite:///%kernel.project_dir%/../var/db.sqlite
 ###< doctrine/doctrine-bundle ###
 
-### records folder
+### records folder and disk
-RECORDS_DIR=%kernel.project_dir%/../var/chunks # adapt to your needs
+RECORDS_DISK=/dev/sda1
+RECORDS_DIR=%kernel.project_dir%/../var/chunks
 ###
 
 ###> symfony/mailer ###

www/composer.lock

@@ -174,26 +174,27 @@
         },
         {
             "name": "doctrine/collections",
-            "version": "1.6.8",
+            "version": "1.7.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/doctrine/collections.git",
-                "reference": "1958a744696c6bb3bb0d28db2611dc11610e78af"
+                "reference": "07d15c8a766e664ec271ae84e5dfc597aeeb03b1"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/doctrine/collections/zipball/1958a744696c6bb3bb0d28db2611dc11610e78af",
+                "url": "https://api.github.com/repos/doctrine/collections/zipball/07d15c8a766e664ec271ae84e5dfc597aeeb03b1",
-                "reference": "1958a744696c6bb3bb0d28db2611dc11610e78af",
+                "reference": "07d15c8a766e664ec271ae84e5dfc597aeeb03b1",
                 "shasum": ""
             },
             "require": {
+                "doctrine/deprecations": "^0.5.3 || ^1",
                 "php": "^7.1.3 || ^8.0"
             },
             "require-dev": {
                 "doctrine/coding-standard": "^9.0",
-                "phpstan/phpstan": "^0.12",
+                "phpstan/phpstan": "^1.4.8",
                 "phpunit/phpunit": "^7.5 || ^8.5 || ^9.1.5",
-                "vimeo/psalm": "^4.2.1"
+                "vimeo/psalm": "^4.22"
             },
             "type": "library",
             "autoload": {
@@ -237,22 +238,22 @@
             ],
             "support": {
                 "issues": "https://github.com/doctrine/collections/issues",
-                "source": "https://github.com/doctrine/collections/tree/1.6.8"
+                "source": "https://github.com/doctrine/collections/tree/1.7.0"
             },
-            "time": "2021-08-10T18:51:53+00:00"
+            "time": "2022-08-18T05:44:45+00:00"
         },
         {
             "name": "doctrine/common",
-            "version": "3.3.0",
+            "version": "3.3.1",
             "source": {
                 "type": "git",
                 "url": "https://github.com/doctrine/common.git",
-                "reference": "c824e95d4c83b7102d8bc60595445a6f7d540f96"
+                "reference": "6a76bd25b1030d35d6ba2bf2f69ca858a41fc580"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/doctrine/common/zipball/c824e95d4c83b7102d8bc60595445a6f7d540f96",
+                "url": "https://api.github.com/repos/doctrine/common/zipball/6a76bd25b1030d35d6ba2bf2f69ca858a41fc580",
-                "reference": "c824e95d4c83b7102d8bc60595445a6f7d540f96",
+                "reference": "6a76bd25b1030d35d6ba2bf2f69ca858a41fc580",
                 "shasum": ""
             },
             "require": {
@@ -261,6 +262,7 @@
             },
             "require-dev": {
                 "doctrine/coding-standard": "^9.0",
+                "doctrine/collections": "^1",
                 "phpstan/phpstan": "^1.4.1",
                 "phpstan/phpstan-phpunit": "^1",
                 "phpunit/phpunit": "^7.5.20 || ^8.5 || ^9.0",
@@ -313,7 +315,7 @@
             ],
             "support": {
                 "issues": "https://github.com/doctrine/common/issues",
-                "source": "https://github.com/doctrine/common/tree/3.3.0"
+                "source": "https://github.com/doctrine/common/tree/3.3.1"
             },
             "funding": [
                 {
@@ -329,20 +331,20 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2022-02-05T18:28:51+00:00"
+            "time": "2022-08-20T10:48:54+00:00"
         },
         {
             "name": "doctrine/dbal",
-            "version": "3.4.0",
+            "version": "3.4.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/doctrine/dbal.git",
-                "reference": "118a360e9437e88d49024f36283c8bcbd76105f5"
+                "reference": "22de295f10edbe00df74f517612f1fbd711131e2"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/doctrine/dbal/zipball/118a360e9437e88d49024f36283c8bcbd76105f5",
+                "url": "https://api.github.com/repos/doctrine/dbal/zipball/22de295f10edbe00df74f517612f1fbd711131e2",
-                "reference": "118a360e9437e88d49024f36283c8bcbd76105f5",
+                "reference": "22de295f10edbe00df74f517612f1fbd711131e2",
                 "shasum": ""
             },
             "require": {
@@ -424,7 +426,7 @@
             ],
             "support": {
                 "issues": "https://github.com/doctrine/dbal/issues",
-                "source": "https://github.com/doctrine/dbal/tree/3.4.0"
+                "source": "https://github.com/doctrine/dbal/tree/3.4.2"
             },
             "funding": [
                 {
@@ -440,7 +442,7 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2022-08-06T20:35:57+00:00"
+            "time": "2022-08-21T14:21:06+00:00"
         },
         {
             "name": "doctrine/deprecations",

www/nginx.conf

@@ -1,34 +0,0 @@
-server {
-	listen 80;
-	server_name birdnet.example.com;
-	root /var/www/html;
-
-	location / {
-		return 302 https://$server_name$request_uri;
-	}
-}
-
-server {
-	listen 443 ssl;
-	server_name birdnet.example.com;
-	root /var/www/html;
-	ssl on;
-	ssl_certificate /etc/nginx/ssl/birdnet.crt;
-	ssl_certificate_key /etc/nginx/ssl/birdnet.key;
-
-	index index.html index.htm index.php;
-
-	location / {
-    		try_files $uri $uri/ /index.php$is_args$args;
-    }
-
-    location ~ \.php$ {
-            fastcgi_split_path_info ^(.+\.php)(/.+)$;
-            fastcgi_pass unix:/run/php-fpm/www.sock;
-            fastcgi_index index.php;
-            include fastcgi.conf;
-    }
-
-	access_log /var/log/nginx/birdnet/birdnet-access.log;
-    error_log  /var/log/nginx/birdnet/birdnet-error.log error;
-}

www/nginx.conf.template

@@ -0,0 +1,66 @@
+server {
+	listen 80;
+	server_name <SERVER_NAME>;
+
+	location / {
+		return 302 https://$host$request_uri;
+	}
+
+    location /.well-known/acme-challenge {
+        alias /var/www/html/.well-known/acme-challenge;
+        allow all;
+    }
+}
+
+server {
+	listen 443 ssl;
+	server_name <SERVER_NAME>;
+
+	fastcgi_buffers  16 16k;
+    fastcgi_buffer_size  32k;	
+
+	root  <SYMFONY_PUBLIC>;
+
+	ssl_certificate  <CERTIFICATE>;
+	ssl_certificate_key  <PRIVATE_KEY>;
+
+	index index.php;
+
+	location / {
+    		try_files $uri $uri/ /index.php$is_args$args;
+    }
+
+	location ^~ /media/records {
+		autoindex on;
+		alias <RECORDS_DIR>;
+	}
+	
+	location ^~ /media/charts {
+		autoindex on;
+		alias <CHARTS_DIR>;
+	}
+
+    location ~ \.php$ {
+		fastcgi_split_path_info ^(.+\.php)(/.+)$;
+		fastcgi_pass unix:/run/php/php8.1-fpm.sock;
+		fastcgi_index index.php;
+		include fastcgi.conf;
+    }
+
+	location /stream {
+		proxy_pass http://localhost:8000/stream;
+    }
+
+	location ~ ^/ttyd(.*)$ {
+    		proxy_http_version 1.1;
+    		proxy_set_header Host $host;
+    		proxy_set_header X-Forwarded-Proto $scheme;
+    		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    		proxy_set_header Upgrade $http_upgrade;
+    		proxy_set_header Connection "upgrade";
+    		proxy_pass http://127.0.0.1:7681/$1;
+	}
+
+	access_log /var/log/nginx/birdnet/birdnet-access.log;
+	error_log  /var/log/nginx/birdnet/birdnet-error.log error;
+}

www/package-lock.json

@@ -16,6 +16,7 @@
                 "core-js": "^3.23.0",
                 "git-revision-webpack-plugin": "^5.0.0",
                 "regenerator-runtime": "^0.13.9",
+                "webpack": "^5.74.0",
                 "webpack-notifier": "^1.15.0"
             }
         },
@@ -7630,7 +7631,6 @@
             "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.74.0.tgz",
             "integrity": "sha512-A2InDwnhhGN4LYctJj6M1JEaGL7Luj6LOmyBHjcI8529cm5p6VXiTIW2sn6ffvEAKmveLzvu4jrihwXtPojlAA==",
             "dev": true,
-            "license": "MIT",
             "dependencies": {
                 "@types/eslint-scope": "^3.7.3",
                 "@types/estree": "^0.0.51",

www/src/Controller/DisksController.php

@@ -10,8 +10,8 @@ class DisksController extends AbstractController
 {
 
     /**
-     * @Route("/disks/", name="disks_index")
+     * @Route("/disks/", name="disks")
-     * @Route("{_locale}/disks/", name="disks_index_i18n")
+     * @Route("{_locale}/disks/", name="disks_i18n")
      */
     public function index() {
         return $this->render('disks/index.html.twig', [

www/src/Controller/HomeController.php

@@ -1,4 +1,5 @@
 <?php
+
 namespace App\Controller;
 
 use Symfony\Component\HttpFoundation\Response;
@@ -6,25 +7,33 @@ use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\Routing\Annotation\Route;
 use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
 use App\AppBundle\ConnectionObservations;
+use Psr\Log\LoggerInterface;
+use Symfony\Component\HttpKernel\Log\Logger;
 
 class HomeController extends AbstractController
 {
     private ConnectionObservations $connection;
+    private LoggerInterface $logger;
 
-    public function __construct(ConnectionObservations $connection)
+    public function __construct(ConnectionObservations $connection, LoggerInterface $logger)
     {
         $this->connection = $connection;
+        $this->logger = $logger;
     }
 
     /**
      * @Route("", name="home")
      * @Route("/{_locale<%app.supported_locales%>}/", name="home_i18n")
      */
-    public function index()
+    public function index(Request $request)
     {
+        $date = $request->get("on");
+        if ($date == null) {
+            $date = date("Y-m-d");
+        }
         return $this->render('index.html.twig', [
-            "stats" => $this->get_stats(),
+            "stats" => $this->get_stats($date),
-            "charts" => $this->last_chart_generated(),
+            "charts" => $this->last_chart_generated($date),
         ]);
     }
 
@@ -34,53 +43,88 @@ class HomeController extends AbstractController
      */
     public function about()
     {
-        return $this->render('about/index.html.twig', [
+        return $this->render('about/index.html.twig', []);
-            
-        ]);
     }
 
-    private function get_stats()
+    private function get_stats($date)
     {
         $stats = array();
         $stats["most-recorded-species"] = $this->get_most_recorded_species();
         $stats["last-detected-species"] = $this->get_last_recorded_species();
+        $stats["number-of-species-detected"] = $this->get_number_of_species_detected($date);
         return $stats;
     }
 
     private function get_most_recorded_species()
     {
+        $species = [];
         $sql = "SELECT `scientific_name`, `common_name`, COUNT(*) AS contact_count 
                 FROM `taxon` 
                 INNER JOIN `observation` 
                 ON `taxon`.`taxon_id` = `observation`.`taxon_id`
                 ORDER BY `contact_count` DESC LIMIT 1";
-        $stmt = $this->connection->prepare($sql);
+        try {
-        $result = $stmt->executeQuery();
+            $stmt = $this->connection->prepare($sql);
-        $species = $result->fetchAllAssociative();
+            $result = $stmt->executeQuery();
-        return $species[0];
+            $species = $result->fetchAllAssociative()[0];
+        } catch (\Exception $e) {
+            $this->logger->error($e->getMessage());
+        }
+        return $species;
     }
 
     private function get_last_recorded_species()
     {
+        $species = [];
         $sql = "SELECT `scientific_name`, `common_name`, `date`, `audio_file`, `confidence`
                 FROM `observation`
                 INNER JOIN `taxon`
                 ON `observation`.`taxon_id` = `taxon`.`taxon_id`
                 ORDER BY `date` DESC LIMIT 1";
-        $stmt = $this->connection->prepare($sql);
+        try {
-        $result = $stmt->executeQuery();
+            $stmt = $this->connection->prepare($sql);
-        $species = $result->fetchAllAssociative();
+            $result = $stmt->executeQuery();    
-        return $species[0];
+            $species = $result->fetchAllAssociative()[0];
+        } catch (\Exception $e) {
+            $this->logger->error($e->getMessage());
+        }
+        return $species;
     }
 
-    private function last_chart_generated() {
+    private function get_number_of_species_detected($date)
+    {
+        $count = 0;
+        $sql = "SELECT COUNT(`taxon_id`) AS contact_count
+                FROM `observation`
+                WHERE STRFTIME('%Y-%m-%d', `date`) = :date
+                GROUP BY `taxon_id`";
+        try {
+            $stmt = $this->connection->prepare($sql);
+            $stmt->bindValue(":date", $date);
+            $result = $stmt->executeQuery();
+            $output = $result->fetchAllAssociative();
+            if ($output != null) {
+                $count = $output[0]["contact_count"];
+            }
+        } catch (\Exception $e) {
+            $this->logger->error($e->getMessage());
+        }
+        return $count;
+    }
 
+    private function last_chart_generated()
+    {
         $files = glob($this->getParameter('kernel.project_dir') . '/../var/charts/*.png');
-        usort($files, function($a, $b) {
+        if (count($files) > 0) {
-            return filemtime($b) - filemtime($a);
+            usort($files, function ($a, $b) {
-        });
+                return filemtime($a) - filemtime($b);
-        $last_chart = basename(array_pop($files));
+            });
-        return $last_chart;
-    }
             
+            $last_chart = basename(array_pop($files));
+            return $last_chart;
+        } else {
+            $this->logger->info("No charts found");
+            return "";
+        }
+    }
 }

www/src/Controller/LogsController.php

@@ -1,39 +0,0 @@
-<?php
-
-namespace App\Controller;
-
-use Symfony\Component\HttpFoundation\Response;
-use Symfony\Component\HttpFoundation\Request;
-use Symfony\Component\Routing\Annotation\Route;
-use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
-
-class LogsController extends AbstractController
-{
-    private $allowed_services = "recording analyzis miner plotter";
-    /**
-     * @Route("/logs/{service}", name="logs")
-     * @Route("/{_locale<%app.supported_locales%>}/logs/{service}", name="logs_i18n")
-     */
-    public function logs($service = "all")
-    {
-        $logs = "";
-        if ($service === "all") {
-            foreach (explode(" ", $this->allowed_services) as $service) {
-                $logs .= $this->journal_logs($service);
-            }
-        } else if (str_contains($this->allowed_services, $service)) {
-            $logs .= $this->journal_logs($service);
-        } else {
-            return new Response("Service not found", Response::HTTP_BAD_REQUEST);
-        }
-        return $this->render('logs/logs.html.twig', [
-            'logs' => $logs
-        ]);
-    }
-
-    private function journal_logs($service)
-    {
-        $logs = shell_exec("journalctl -u birdnet_recording -n 10");
-        return $logs;
-    }
-}

www/src/Controller/TodayController.php

@@ -6,15 +6,20 @@ use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\Routing\Annotation\Route;
 use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
 use App\AppBundle\ConnectionObservations;
+use Psr\Log\LoggerInterface;
 
 class TodayController extends AbstractController
-{    private ConnectionObservations $connection;
+{    
+    private ConnectionObservations $connection;
+    private LoggerInterface $logger;
 
-    public function __construct(ConnectionObservations $connection)
+    public function __construct(ConnectionObservations $connection, LoggerInterface $logger)
     {
         $this->connection = $connection;
+        $this->logger = $logger;
     }
 
+
     /**
      * @Route("/today", name="today")
      * @Route("/{_locale<%app.supported_locales%>}/today", name="today_i18n")
@@ -88,28 +93,38 @@ class TodayController extends AbstractController
 
     private function recorded_species_by_date($date)
     {
+        $recorded_species = [];
         $sql = "SELECT `taxon`.`taxon_id`, `scientific_name`, `common_name`, COUNT(*) AS `contact_count`, MAX(`confidence`) AS max_confidence
                 FROM observation 
                 INNER JOIN taxon 
                 ON observation.taxon_id = taxon.taxon_id 
                 WHERE strftime('%Y-%m-%d', `observation`.`date`) = :date 
                 GROUP BY observation.taxon_id";
-        $stmt = $this->connection->prepare($sql);
+        try {
-        $stmt->bindValue(':date', $date);
+            $stmt = $this->connection->prepare($sql);
-        $result = $stmt->executeQuery();
+            $stmt->bindValue(':date', $date);
-        return $result->fetchAllAssociative();
+            $result = $stmt->executeQuery();
+            $recorded_species = $result->fetchAllAssociative();
+        } catch (\Exception $e) {
+            $this->logger->error($e->getMessage());
+        }
+        return $recorded_species;
     }
 
     private function recorded_species_by_id_and_date($id, $date)
     {
         /* Get taxon even if there is no record this date */
         $sql = "SELECT * FROM `taxon` WHERE `taxon_id` = :id";
-        $stmt = $this->connection->prepare($sql);
+        $taxon = [];
-        $stmt->bindValue(':id', $id);
+        $stat = [];
-        $result = $stmt->executeQuery();
+        $records = [];
-        $taxon = $result->fetchAllAssociative()[0];
+        try {
-        if (!$taxon) {
+            $stmt = $this->connection->prepare($sql);
-            return [];
+            $stmt->bindValue(':id', $id);
+            $result = $stmt->executeQuery();
+            $taxon = $result->fetchAllAssociative()[0];
+        } catch (\Exception $e) {
+            $this->logger->error($e->getMessage());
         }
         /* Get daily stats */
         $sql = "SELECT COUNT(*) AS `contact_count`, MAX(`confidence`) AS `max_confidence` 
@@ -118,33 +133,47 @@ class TodayController extends AbstractController
                 ON `taxon`.`taxon_id` = `observation`.`taxon_id` 
                 WHERE strftime('%Y-%m-%d', `observation`.`date`) = :date
                 AND `observation`.`taxon_id` = :id";
-        $stmt = $this->connection->prepare($sql);
+        try {
-        $stmt->bindValue(':id', $id);
+            $stmt = $this->connection->prepare($sql);
-        $stmt->bindValue(':date', $date);
+            $stmt->bindValue(':id', $id);
-        $result = $stmt->executeQuery();
+            $stmt->bindValue(':date', $date);
-        $stat = $result->fetchAllAssociative();
+            $result = $stmt->executeQuery();
+            $stat = $result->fetchAllAssociative();
+        } catch (\Exception $e) {
+            $this->logger->error($e->getMessage());
+        }
         $sql = "SELECT * FROM `observation` 
                 WHERE `taxon_id` = :id 
                 AND strftime('%Y-%m-%d', `observation`.`date`) = :date
                 ORDER BY `observation`.`date` ASC";
-        $stmt = $this->connection->prepare($sql);
+        try {
-        $stmt->bindValue(':id', $id);
+            $stmt = $this->connection->prepare($sql);
-        $stmt->bindValue(':date', $date);
+            $stmt->bindValue(':id', $id);
-        $result = $stmt->executeQuery();
+            $stmt->bindValue(':date', $date);
-        $records = $result->fetchAllAssociative();
+            $result = $stmt->executeQuery();
+            $records = $result->fetchAllAssociative();
+        } catch (\Exception $e) {
+            $this->logger->error($e->getMessage());
+        }
         return array("taxon" => $taxon, "stat" => $stat, "records" => $records);
     }
 
     private function best_confidence_today($id, $date)
     {
+        $best_confidence = 0;
         $sql = "SELECT MAX(`confidence`) AS confidence 
                 FROM `observation` 
                 WHERE strftime('%Y-%m-%d', `observation`.`date`) = :date 
                 AND `taxon_id` = :id";
-        $stmt = $this->connection->prepare($sql);
+        try {
-        $stmt->bindValue(':id', $id);
+            $stmt = $this->connection->prepare($sql);
-        $stmt->bindValue(':date', $date);
+            $stmt->bindValue(':id', $id);
-        $result = $stmt->executeQuery();
+            $stmt->bindValue(':date', $date);
-        return $result->fetchAllAssociative();
+            $result = $stmt->executeQuery();
+            $result->fetchAllAssociative();
+        } catch (\Exception $e) {
+            $this->logger->error($e->getMessage());
+        }
+        return $best_confidence;
     }
 }

www/templates/disks/index.html.twig

@@ -1,3 +1,4 @@
+{% extends "base.html.twig" %}
 {% block content %}
     <h2>{{ "Disk usage"|trans }}</h2>
     <div class="disk">

www/templates/menu.html.twig

@@ -38,14 +38,22 @@
           } %}
         </ul>
       </li>
-      {% include 'utils/nav-item.html.twig' with {
+      <li class="dropdown">
-        route: 'logs',
+        <span class="dropdown-toggle">{{ "Tools"|trans }}</span>
-        text: 'View Logs'|trans
+        <ul class="dropdown-content">
-      } %}
+          <li><a href="/ttyd">
-      {% include 'utils/nav-item.html.twig' with {
+            {{ "Logs"|trans }}
-        route: 'services_status',
+          </a></li>
-        text: 'Status'|trans
+          {% include 'utils/nav-item.html.twig' with {
-      } %}
+            route: 'services_status',
+            text: 'Status'|trans
+          } %}
+          {% include "utils/nav-item.html.twig" with {
+            route: 'disks',
+            text: 'Disks'|trans
+          } %}
+        </ul>
+      </li>
     </ul>
   </div>
 </nav>

www/templates/stats.html.twig

@@ -1,50 +1,78 @@
 <div id="stats">
-    <h2>{{ "Quick Stats" | trans }}</h2>
+  <h2>
-    <ul>
+    {{ 'Quick Stats'|trans }}
-        <li class="most-recorded-species">
+  </h2>
-            {{ "Most recorded species" | trans }}:
+  <ul>
-            {% if stats["most-recorded-species"] is defined %}
+    <li class="stat">
-            <span class="scientific-name">
+      {{ 'Most recorded species'|trans }}:{% if
-                {{ stats["most-recorded-species"]["scientific_name"] }}
+        stats['most-recorded-species'] is defined
-            </span>
+          and (stats['most-recorded-species']|length) > 0 %}
-            (<span class="common_name">{{ stats["most-recorded-species"]["common_name"] }}</span>)
+        <span class="scientific-name">
-            {{ "with" | trans }}
+          {{ stats['most-recorded-species']['scientific_name'] }}
-            <span class="observation-count">
+        </span>
-                {{ stats["most-recorded-species"]["contact_count"] }}
+        (<span class="common_name">
-            </span>
+          {{ stats['most-recorded-species']['common_name'] }}
-            {{ "contacts" | trans }}.
+        </span>)
-            {% else %}
+        {{ 'with'|trans }}
-                {{ "No species in database." | trans }}
+        <span class="observation-count">
-            {% endif %}
+          {{ stats['most-recorded-species']['contact_count'] }}
-        </li>
+        </span>
-        <li class="last-recorded-species">
+        {{ 'contacts'|trans }}.
-            {{ "Last detected species" | trans }}:
+      {% else %}
-            {% if stats["last-detected-species"] is defined %}
+        {{ 'No species in database.'|trans }}
-            <span class="scientific-name">
+      {% endif %}
-                {{ stats["last-detected-species"]["scientific_name"] }}
+    </li>
-            </span>
+    <li class="stat">
-            (<span class="common_name">{{ stats["last-detected-species"]["common_name"] }}</span>)
+      {{ 'Last detected species'|trans }}:{% if
-            {{ "with" | trans }}
+        stats['last-detected-species'] is defined
-            <span class="confidence">
+          and (stats['last-detected-species']|length) > 0 %}
-                {{ stats["last-detected-species"]["confidence"] }}
+        <span class="scientific-name">
-            </span>
+          {{ stats['last-detected-species']['scientific_name'] }}
-            {{ "AI confidence" | trans }}
+        </span>
-            <span class="datetime">
+        (<span class="common_name">
-            {% set date = stats["last-detected-species"]["date"] %}
+          {{ stats['last-detected-species']['common_name'] }}
-            {% if  date | date("Y-m-d") == "now" | date("Y-m-d") %}
+        </span>)
-                {{ "today" | trans }}
+        {{ 'with'|trans }}
-            {% else %}
+        <span class="confidence">
-                {{ "on" | trans }}
+          {{ stats['last-detected-species']['confidence'] }}
-                {{ date | format_datetime("full", "none") }}
+        </span>
-            {% endif %}
+        {{ 'AI confidence'|trans }}
-            at 
+        <span class="datetime">
-            <span class="time">
+          {% set date = stats['last-detected-species']['date'] %}
-                {{ date | date("H:i") }} 
+          {% if (date|date('Y-m-d')) == ('now'|date('Y-m-d')) %}
-            </span>
+            {{ 'today'|trans }}
-            </span>.
+          {% else %}
-            {% else %}
+            {{ 'on'|trans }}
-                {{ "No species in database" | trans }}
+            {{ date|format_datetime('full', 'none') }}
-            {% endif %}
+          {% endif %}at
-        </li>
+          <span class="time">{{ date|date('H:i') }}</span>
-    </ul>
+        </span>.
+      {% else %}
+        {{ 'No species in database'|trans }}
+      {% endif %}
+    </li>
+    <li class="stat">
+      {% set today = 'now'|date('Y-m-d') %}
+      {% set date = app.request.get('on') %}
+      {% if
+        stats['number-of-species-detected'] is defined
+          and stats['number-of-species-detected'] > 0 %}
+        {% if today == date %}
+          {{ 'Number of species detected today: '|trans }}
+        {% else %}
+          {{ 'Number of species detected on '|trans }}
+          {{ date|format_datetime('full', 'none') }}:
+        {% endif %}
+        <span>{{ stats['number-of-species-detected'] }}</span>.
+      {% else %}
+        {# {{ 'No species detected today'|trans }} #}
+        {% if today == date %}
+          {{ 'No species detected today.'|trans }}
+        {% else %}
+          {{ 'No species detected on '|trans }}
+          {{ date|format_datetime('full', 'none') }}
+        {% endif %}
+      {% endif %}
+    </li>
+  </ul>
 </div>

www/translations/messages+intl-icu.fr.xlf

@@ -11,7 +11,7 @@
       </trans-unit>
       <trans-unit id="VAw_dLX" resname="BirdNET-stream is a realtime soundscape analyzis software powered by BirdNET AI.">
         <source>BirdNET-stream is a realtime soundscape analyzis software powered by BirdNET AI.</source>
-        <target>BirdNET-stream est un logiciel d'analyse en temps réel de l'environement sonore basé sur BirdNET.</target>
+        <target>BirdNET-stream est un logiciel d'analyse en temps réel de l'environnement sonore basé sur BirdNET.</target>
       </trans-unit>
       <trans-unit id="vvz1r3A" resname="It aims to be able to run on any computer with a microphone.">
         <source>It aims to be able to run on any computer with a microphone.</source>