From cf032c5f8d0bda84148e81196df24f7ca7071f91 Mon Sep 17 00:00:00 2001 From: Samuel ORTION Date: Sun, 21 Aug 2022 06:13:15 +0200 Subject: [PATCH] Update install.sh --- .dockerignore | 4 + .ideas/birdnet_archive.sh | 133 ++++++++++++++++++ .ideas/fill_taxa.sh | 2 +- .ideas/insert_observation.sh | 2 +- INSTALL.md | 2 +- TODO | 4 +- daemon/birdnet_analyzis.sh | 2 +- daemon/birdnet_clean.sh | 2 +- daemon/birdnet_manager.sh | 2 +- daemon/birdnet_miner.sh | 2 +- daemon/birdnet_recording.sh | 24 ++-- daemon/birdnet_streaming.sh | 2 +- daemon/database/migrations/migrate.sh | 2 +- daemon/database/scripts/create.sh | 2 +- daemon/database/scripts/database.py | 2 +- daemon/database/scripts/database.sh | 4 +- daemon/plotter/chart.py | 158 +++++++++------------- docker/all/Dockerfile | 10 +- docker/all/README.md | 2 +- install.sh | 105 +++++++++----- utils/fix_permissions.sh | 11 +- www/package-lock.json | 2 +- www/package.json | 1 + www/src/Controller/DisksController.php | 4 +- www/templates/disks/index.html.twig | 1 + www/templates/menu.html.twig | 25 ++-- www/translations/messages+intl-icu.fr.xlf | 2 +- www/yarn.lock | 2 +- 28 files changed, 333 insertions(+), 181 deletions(-) create mode 100644 .dockerignore create mode 100755 .ideas/birdnet_archive.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4585e05 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +var +.venv +.github +.ideas diff --git a/.ideas/birdnet_archive.sh b/.ideas/birdnet_archive.sh new file mode 100755 index 0000000..cf18b50 --- /dev/null +++ b/.ideas/birdnet_archive.sh @@ -0,0 +1,133 @@ +#! /usr/bin/env bash +# Compress wav to flac and archive them as zip + +# Requires: tar, gzip, ffmpeg + +set -e + +DEBUG=${DEBUG:-0} +debug() { + [[ $DEBUG -eq 1 ]] && echo "$@" +} +error() { + echo 1>&2 "$@" +} + +audio_compress() { + local filepath + filepath="$1" + if [[ "$DRY" -eq 1 ]]; then + debug "Would compress $filepath to flac" + return 0 + else + debug "Compressing $filepath" + ffmpeg -i "$filepath" -acodec flac -compression_level 10 "${filepath%.wav}.flac" + fi +} + +all_audio_compress() { + local dir + dir="$1" + debug "Compressing all .wav audio in $dir" + for filepath in "$dir"/*.wav; do + if [[ "$DRY" -eq 1 ]]; then + debug "Would convert $filepath to flac and remove it" + else + audio_compress "$filepath" + debug "Removing $filepath" + rm "$filepath" + fi + done +} + +record_datetime() { + source_wav=$1 + source_base=$(basename "$source_wav" ".wav") + record_date=$(echo "$source_base" | cut -d"_" -f2) + record_time=$(echo "$source_base" | cut -d"_" -f3) + YYYY=$(echo "$record_date" | cut -c 1-4) + MM=$(echo "$record_date" | cut -c 5-6) + DD=$(echo "$record_date" | cut -c 7-8) + HH=$(echo "$record_time" | cut -c 1-2) + MI=$(echo "$record_time" | cut -c 3-4) + SS=$(echo "$record_time" | cut -c 5-6) + SSS="000" + date="$YYYY-$MM-$DD $HH:$MI:$SS.$SSS" + echo "$date" +} + +source_wav() { + model_output_dir="$1" + wav=$(basename "$model_output_dir" | rev | cut --complement -d"." -f1 | rev) + echo "$wav" +} + +birdnet_archive_older_than() { + local days + days="$1" + local date + date=$(date +"%Y-%m-%d") + local date_pivot + date_pivot=$(date -d "$date + $days days" +"%Y-%m-%d") + move_records_to_archive "$date_pivot" + zip_archives +} + +move_records_to_archive() { + local date + date="$1" + local archives_dir + archives_dir="$2" + archive_path="${ARCHIVE_DIR}/$date" + debug "Moving records from $CHUNK_FOLDER/out to $archives_path" + for filepath in $(find "$CHUNK_FOLDER/out/" -name '*.wav.d'); do + wav=$(source_wav "$filepath") + dir=$(dirname "$filepath") + record_datetime=$(record_datetime "$wav") + if [[ "$record_datetime" == "$date" ]]; then + debug "Moving $filepath to $archive_path" + if [[ ! -d "$archive_path" ]]; then + mkdir -p "$archive_path" + fi + mv "$filepath" "$archive_path" + debug "Moving model output directory to archive" + mv "$dir" "$archive_path/" + debug "Moving wav to archive" + mv "$CHUNK_FOLDER/out/$wav" "$archive_path/" + fi + done +} + +zip_archives() { + debug "Zipping archives in ${ARCHIVE_DIR}" + for archive_path in $(find "${ARCHIVE_DIR}" -type d); do + archive_name="birdnet_$(basename "$archive_path" | tr '-' '').tar.gz" + if [[ "$DRY" -eq 1 ]]; then + debug "Would zip $archive_path to $archive_name" + else + debug "Zipping $archive_path to $archive_name" + tar -czf "$archive_name" -C "$archive_path" . + debug "Removing temporary archive folder in ${ARCHIVE_DIR}" + rm -rf "$archive_path" + fi + done +} + +main() { + config_filepath="./config/birdnet.conf" + [ -f "$config_filepath" ] || { + error "Config file not found: $config_filepath" + exit 1 + } + source "$config_filepath" + if [[ -z "CHUNK_FOLDER" ]]; then + error "CHUNK_FOLDER not set in config file" + exit 1 + fi + if [[ -z "ARCHIVE_FOLDER" ]]; then + error "ARCHIVE_FOLDER not set in config file" + exit 1 + fi + debug "Launch birdnet archive script from $CHUNK_FOLDER to $ARCHIVE_FOLDER" + birdnet_archive_older_than $DAYS_TO_KEEP +} diff --git a/.ideas/fill_taxa.sh b/.ideas/fill_taxa.sh index b0348fe..58b48e4 100755 --- a/.ideas/fill_taxa.sh +++ b/.ideas/fill_taxa.sh @@ -3,7 +3,7 @@ set -e # Load config file -config_filepath="./config/analyzer.conf" +config_filepath="./config/birdnet.conf" if [ -f "$config_filepath" ]; then source "$config_filepath" diff --git a/.ideas/insert_observation.sh b/.ideas/insert_observation.sh index 73123d9..79aafa3 100644 --- a/.ideas/insert_observation.sh +++ b/.ideas/insert_observation.sh @@ -1,7 +1,7 @@ #! /usr/bin/env bash # Load config file -config_filepath="./config/analyzer.conf" +config_filepath="./config/birdnet.conf" if [ -f "$config_filepath" ]; then source "$config_filepath" diff --git a/INSTALL.md b/INSTALL.md index 1100159..22f4239 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -170,7 +170,7 @@ Launch and enable icecast: sudo systemctl enable --now icecast2 ``` -Adapt `config/analyzer.conf` to this configuration: +Adapt `config/birdnet.conf` to this configuration: ```conf ICECAST_USER=source diff --git a/TODO b/TODO index d9235ab..ea34237 100644 --- a/TODO +++ b/TODO @@ -1,5 +1,5 @@ -- Fix clean script - Fix service manager - Add docker support - Species i18n -- File purge policy \ No newline at end of file +- File purge policy +- Add and test RTSP support \ No newline at end of file diff --git a/daemon/birdnet_analyzis.sh b/daemon/birdnet_analyzis.sh index 3986134..8943faf 100755 --- a/daemon/birdnet_analyzis.sh +++ b/daemon/birdnet_analyzis.sh @@ -8,7 +8,7 @@ debug() { fi } -config_filepath="./config/analyzer.conf" +config_filepath="./config/birdnet.conf" if [ -f "$config_filepath" ]; then source "$config_filepath" diff --git a/daemon/birdnet_clean.sh b/daemon/birdnet_clean.sh index d75c8dc..dc2345a 100755 --- a/daemon/birdnet_clean.sh +++ b/daemon/birdnet_clean.sh @@ -13,7 +13,7 @@ debug() { fi } -config_filepath="./config/analyzer.conf" +config_filepath="./config/birdnet.conf" if [ -f "$config_filepath" ]; then source "$config_filepath" diff --git a/daemon/birdnet_manager.sh b/daemon/birdnet_manager.sh index 2fbe371..3ec21ec 100755 --- a/daemon/birdnet_manager.sh +++ b/daemon/birdnet_manager.sh @@ -3,7 +3,7 @@ set -e # set -x -config_filepath="./config/analyzer.conf" +config_filepath="./config/birdnet.conf" if [ -f "$config_filepath" ]; then source "$config_filepath" diff --git a/daemon/birdnet_miner.sh b/daemon/birdnet_miner.sh index f10c4dd..d03075d 100755 --- a/daemon/birdnet_miner.sh +++ b/daemon/birdnet_miner.sh @@ -16,7 +16,7 @@ debug() { source ./daemon/database/scripts/database.sh # Load config -source ./config/analyzer.conf +source ./config/birdnet.conf # Check config if [[ -z ${CHUNK_FOLDER} ]]; then echo "CHUNK_FOLDER is not set" diff --git a/daemon/birdnet_recording.sh b/daemon/birdnet_recording.sh index 0a16df9..7b002b9 100755 --- a/daemon/birdnet_recording.sh +++ b/daemon/birdnet_recording.sh @@ -1,6 +1,6 @@ #! /usr/bin/env bash -DEBUG=${DEBUG:-0} +DEBUG=${DEBUG:-1} export PULSE_RUNTIME_PATH="/run/user/$(id -u)/pulse/" @@ -23,7 +23,7 @@ record_loop() { DURATION=$2 debug "New recording loop." while true; do - record $DEVICE $DURATION + record_device $DEVICE $DURATION done } @@ -33,7 +33,7 @@ record_stream() { local STREAM=$1 local DURATION=$2 local debug "Recording from $STREAM for $DURATION seconds" - ffmpeg $FFMPEG_OPTIONS -f -i ${DEVICE} -t ${DURATION} file:${CHUNK_FOLDER}/in/birdnet_$(date "+%Y%m%d_%H%M%S").wav + ffmpeg $FFMPEG_OPTIONS -f -i ${DEVICE} -t ${DURATION} file:${CHUNK_FOLDER}/in/birdnet_$(date "+%Y%m%d_%H%M%S").wav } record_device() { @@ -43,7 +43,7 @@ record_device() { ffmpeg $FFMPEG_OPTIONS -f pulse -i ${DEVICE} -t ${DURATION} -af "volume=$RECORDING_AMPLIFY" file:${CHUNK_FOLDER}/in/birdnet_$(date "+%Y%m%d_%H%M%S").wav } -config_filepath="./config/analyzer.conf" +config_filepath="./config/birdnet.conf" if [ -f "$config_filepath" ]; then source "$config_filepath" @@ -56,18 +56,18 @@ check_folder [ -z $RECORDING_DURATION ] && RECORDING_DURATION=15 -if [[ -z $AUDIO_DEVICE ]]; then - echo "AUDIO_DEVICE is not set" - exit 1 -fi - if [[ $AUDIO_RECORDING = "true" ]]; then - record_loop $AUDIO_DEVICE $RECORDING_DURATION & + debug "Recording with on board device" + if [[ -z $AUDIO_DEVICE ]]; then + echo "AUDIO_DEVICE is not set" + exit 1 + fi + record_loop $AUDIO_DEVICE $RECORDING_DURATION fi if [[ $AUDIO_STATIONS = "true" ]]; then for station in $(ls ./config/stations/*.conf); do source $station - record_stream $STATION_URL $RECORDING_DURATION & + record_stream $STATION_URL $RECORDING_DURATION done -fi \ No newline at end of file +fi diff --git a/daemon/birdnet_streaming.sh b/daemon/birdnet_streaming.sh index 1b7a3b6..536f0ca 100755 --- a/daemon/birdnet_streaming.sh +++ b/daemon/birdnet_streaming.sh @@ -18,7 +18,7 @@ stream() { -f mp3 "icecast://source:${ICECAST_PASSWORD}@${ICECAST_HOST}:${ICECAST_PORT}/${ICECAST_MOUNT}" -listen 1 } -config_filepath="./config/analyzer.conf" +config_filepath="./config/birdnet.conf" if [ -f "$config_filepath" ]; then source "$config_filepath" diff --git a/daemon/database/migrations/migrate.sh b/daemon/database/migrations/migrate.sh index 615f344..da090be 100755 --- a/daemon/database/migrations/migrate.sh +++ b/daemon/database/migrations/migrate.sh @@ -8,7 +8,7 @@ debug() { } # Load config file -config_filepath="./config/analyzer.conf" +config_filepath="./config/birdnet.conf" if [ -f "$config_filepath" ]; then source "$config_filepath" diff --git a/daemon/database/scripts/create.sh b/daemon/database/scripts/create.sh index 943e910..bb9359f 100755 --- a/daemon/database/scripts/create.sh +++ b/daemon/database/scripts/create.sh @@ -1,7 +1,7 @@ #! /usr/bin/env bash # Load config file -config_filepath="./config/analyzer.conf" +config_filepath="./config/birdnet.conf" if [ -f "$config_filepath" ]; then source "$config_filepath" diff --git a/daemon/database/scripts/database.py b/daemon/database/scripts/database.py index b56ab9e..2f55ed5 100755 --- a/daemon/database/scripts/database.py +++ b/daemon/database/scripts/database.py @@ -7,7 +7,7 @@ verbose = False """Load config""" def load_conf(): - with open("./config/analyzer.conf", "r") as f: + with open("./config/birdnet.conf", "r") as f: conf = f.readlines() res = dict(map(str.strip, sub.split('=', 1)) for sub in conf if '=' in sub) return res diff --git a/daemon/database/scripts/database.sh b/daemon/database/scripts/database.sh index b2be1f2..66c8b08 100755 --- a/daemon/database/scripts/database.sh +++ b/daemon/database/scripts/database.sh @@ -3,7 +3,7 @@ set -e -source ./config/analyzer.conf +source ./config/birdnet.conf # Create database in case it was not created yet ./daemon/database/scripts/create.sh @@ -23,7 +23,7 @@ get_taxon_id() { } insert_taxon() { - query $DATABASE "INSERT INTO taxon (scientific_name, common_name) VALUES (\"$1\", \"$2\")" + query "INSERT INTO taxon (scientific_name, common_name) VALUES (\"$1\", \"$2\")" } insert_location() { diff --git a/daemon/plotter/chart.py b/daemon/plotter/chart.py index 3403ed4..05a7896 100755 --- a/daemon/plotter/chart.py +++ b/daemon/plotter/chart.py @@ -1,8 +1,6 @@ #! /usr/bin/env python3 -from curses import def_prog_mode import sqlite3 -from xml.sax.handler import feature_external_ges import pandas as pd import matplotlib.pyplot as plt from matplotlib.colors import LogNorm @@ -12,109 +10,83 @@ from datetime import datetime CONFIG = { "readings": 10, "palette": "Greens", + "db": "./var/db.sqlite", + "date": "2022-08-14" } -db = None -def get_database(): - global db - if db is None: - db = sqlite3.connect('/home/ortion/Desktop/db.sqlite') - return db +db = sqlite3.connect(CONFIG['db']) +df = pd.read_sql_query("""SELECT common_name, date, location_id, confidence + FROM observation + INNER JOIN taxon + ON observation.taxon_id = taxon.taxon_id""", db) +df['date'] = pd.to_datetime(df['date']) +df['hour'] = df['date'].dt.hour +df['date'] = df['date'].dt.date +df['date'] = df['date'].astype(str) +df_on_date = df[df['date'] == CONFIG['date']] + +top_on_date = (df_on_date['common_name'].value_counts()[:CONFIG['readings']]) -def get_detection_hourly(date): - db = get_database() - df = pd.read_sql_query("""SELECT common_name, date, location_id, confidence - FROM observation - INNER JOIN taxon - ON observation.taxon_id = taxon.taxon_id""", db) +df_top_on_date = df_on_date[df_on_date['common_name'].isin(top_on_date.index)] - df['date'] = pd.to_datetime(df['date']) - df['hour'] = df['date'].dt.hour - df['date'] = df['date'].dt.date - df['date'] = df['date'].astype(str) +# Create a figure with 2 subplots +fig, axs = plt.subplots(1, 2, figsize=(15, 4), gridspec_kw=dict( + width_ratios=[3, 6])) +plt.subplots_adjust(left=None, bottom=None, right=None, + top=None, wspace=0, hspace=0) - df_on_date = df[df['date'] == date] - return df_on_date +# Get species frequencies +frequencies_order = pd.value_counts(df['common_name']).iloc[:CONFIG['readings']].index +# Get min max confidences +confidence_minmax = df_top_on_date.groupby('common_name')['confidence'].max() +confidence_minmax = confidence_minmax.reindex(frequencies_order) +# Norm values for color palette +norm = plt.Normalize(confidence_minmax.values.min(), + confidence_minmax.values.max()) +colors = plt.cm.Greens(norm(confidence_minmax)) +plot = sns.countplot(y='common_name', data=df_top_on_date, palette=colors, order=frequencies_order, ax=axs[0]) -def get_top_species(df, limit=10): - return df['common_name'].value_counts()[:CONFIG['readings']] +plot.set(ylabel=None) +plot.set(xlabel="Detections") +heat = pd.crosstab(df_top_on_date['common_name'], df_top_on_date['hour']) +# Order heatmap Birds by frequency of occurrance +heat.index = pd.CategoricalIndex(heat.index, categories=frequencies_order) +heat.sort_index(level=0, inplace=True) -def get_top_detections(df, limit=10): - df_top_species = get_top_species(df, limit=limit) - return df[df['common_name'].isin(df_top_species.index)] +hours_in_day = pd.Series(data=range(0, 24)) +heat_frame = pd.DataFrame(data=0, index=heat.index, columns=hours_in_day) +heat = (heat + heat_frame).fillna(0) +# Generate heatmap plot +plot = sns.heatmap( + heat, + norm=LogNorm(), + annot=True, + annot_kws={ + "fontsize": 7 + }, + fmt="g", + cmap=CONFIG['palette'], + square=False, + cbar=False, + linewidth=0.5, + linecolor="Grey", + ax=axs[1], + yticklabels=False) +plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7) -def get_frequence_order(df, limit=10): - pd.value_counts(df['common_name']).iloc[:limit] +for _, spine in plot.spines.items(): + spine.set_visible(True) -def presence_chart(date, filename): - df_detections = get_detection_hourly(date) - df_top_detections = get_top_detections(df_detections, limit=CONFIG['readings']) - fig, axs = plt.subplots(1, 2, figsize=(15, 4), gridspec_kw=dict( - width_ratios=[3, 6])) - plt.subplots_adjust(left=None, bottom=None, right=None, - top=None, wspace=0, hspace=0) +plot.set(ylabel=None) +plot.set(xlabel="Hour of day") +fig.subplots_adjust(top=0.9) +plt.suptitle(f"Top {CONFIG['readings']} species (Updated on {datetime.now().strftime('%Y/%m-%d %H:%M')})") - frequencies_order = get_frequence_order(df_detections, limit=CONFIG["readings"]) - # Get min max confidences - confidence_minmax = df_detections.groupby('common_name')['confidence'].max() - # Norm values for color palette - norm = plt.Normalize(confidence_minmax.values.min(), - confidence_minmax.values.max()) - colors = plt.cm.Greens(norm(confidence_minmax)) - plot = sns.countplot(y='common_name', data=df_top_detections, palette=colors, order=frequencies_order, ax=axs[0]) +plt.savefig(f"./var/charts/chart_{CONFIG['date']}.png", dpi=300) +plt.close() - plot.set(ylabel=None) - plot.set(xlabel="Detections") - - heat = pd.crosstab(df_top_detections['common_name'], df_top_detections['hour']) - # Order heatmap Birds by frequency of occurrance - heat.index = pd.CategoricalIndex(heat.index, categories=frequencies_order) - heat.sort_index(level=0, inplace=True) - - hours_in_day = pd.Series(data=range(0, 24)) - heat_frame = pd.DataFrame(data=0, index=heat.index, columns=hours_in_day) - heat = (heat + heat_frame).fillna(0) - - # Generate heatmap plot - plot = sns.heatmap( - heat, - norm=LogNorm(), - annot=True, - annot_kws={ - "fontsize": 7 - }, - fmt="g", - cmap=CONFIG['palette'], - square=False, - cbar=False, - linewidth=0.5, - linecolor="Grey", - ax=axs[1], - yticklabels=False - ) - plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7) - - for _, spine in plot.spines.items(): - spine.set_visible(True) - - plot.set(ylabel=None) - plot.set(xlabel="Hour of day") - fig.subplots_adjust(top=0.9) - plt.suptitle(f"Top {CONFIG['readings']} species (Updated on {datetime.now().strftime('%Y/%m-%d %H:%M')})") - - plt.savefig(filename) - plt.close() - -def main(): - date = datetime.now().strftime('%Y%m%d') - presence_chart(date, f'./var/charts/chart_{date}.png') - # print(get_top_detections(get_detection_hourly(date), limit=10)) - if not db is None: - db.close() - -if __name__ == "__main__": - main() +db.close() \ No newline at end of file diff --git a/docker/all/Dockerfile b/docker/all/Dockerfile index d39f512..8cf0570 100644 --- a/docker/all/Dockerfile +++ b/docker/all/Dockerfile @@ -10,11 +10,13 @@ WORKDIR /home/birdnet # Upgrade system RUN apt-get update && apt-get upgrade -y -# Install curl +# Install sudo RUN apt-get install -y \ - curl \ - sudo + sudo \ + git -RUN curl -sL https://raw.githubusercontent.com/UncleSamulus/BirdNET-stream/master/install.sh | bash +COPY ./install.sh install.sh + +RUN bash ./install.sh USER birdnet diff --git a/docker/all/README.md b/docker/all/README.md index 1805f9f..4d4a7ef 100644 --- a/docker/all/README.md +++ b/docker/all/README.md @@ -9,7 +9,7 @@ ```bash git clone https://github.com/UncleSamulus/BirdNET-stream.git cd ./BirdNET-stream/docker/all -docker build -t "birdnet_all:latest" . +docker build -t "birdnet_all:latest" -f ./docker/all/Dockerfile . ``` If `docker` command does not work because of unsufficient permissions, you could add your user to `docker` group: diff --git a/install.sh b/install.sh index 29434b1..a4a0669 100755 --- a/install.sh +++ b/install.sh @@ -5,13 +5,13 @@ set -e DEBUG=${DEBUG:-0} -REQUIREMENTS="git ffmpeg python3-pip python3-dev" +REQUIREMENTS="git ffmpeg python3 python3-pip python3-dev" REPOSITORY=${REPOSITORY:-https://github.com/UncleSamulus/BirdNET-stream.git} +WORKDIR="$(pwd)/BirdNET-stream" +PYTHON_VENV="./.venv/birdnet-stream" debug() { - if [ $DEBUG -eq 1 ]; then - echo "$1" - fi + [[ $DEBUG -eq 1 ]] && echo "$@" } install_requirements() { @@ -19,52 +19,53 @@ install_requirements() { # Install requirements missing_requirements="" for requirement in $requirements; do - if ! dpkg -s $requirement >/dev/null 2>&1; then + if ! dpkg -s "$requirement" >/dev/null 2>&1; then missing_requirements="$missing_requirements $requirement" fi done - if [ -n "$missing_requirements" ]; then + if [[ -n "$missing_requirements" ]]; then debug "Installing missing requirements: $missing_requirements" - sudo apt-get install -y $missing_requirements + sudo apt-get install -y "$missing_requirements" fi } # Install BirdNET-stream install_birdnetstream() { # Check if repo is not already installed - workdir=$(pwd) - if [ -d "$workdir/BirdNET-stream" ]; then - debug "BirdNET-stream is already installed" + if [[ -d "$DIR" ]]; then + debug "BirdNET-stream is already installed, use update script (not implemented yet)" + exit 1 else + debug "Installing BirdNET-stream" + debug "Creating BirdNET-stream directory" + mkdir -p "$WORKDIR" # Clone BirdNET-stream + cd "$WORKDIR" debug "Cloning BirdNET-stream from $REPOSITORY" - git clone --recurse-submodules $REPOSITORY - # Install BirdNET-stream + git clone --recurse-submodules "$REPOSITORY" . + debug "Creating python3 virtual environment $PYTHON_VENV" + python3 -m venv $PYTHON_VENV + debug "Activating $PYTHON_VENV" + source "$PYTHON_VENV/bin/activate" + debug "Installing python packages" + pip3 install -U pip + pip3 install -r requirements.txt + debug "Creating ./var directory" + mkdir -p ./var/{charts,chunks/{in,out}} fi - cd BirdNET-stream - debug "Creating python3 virtual environment '$PYTHON_VENV'" - python3 -m venv $PYTHON_VENV - debug "Activating $PYTHON_VENV" - source .venv/birdnet-stream/bin/activate - debug "Installing python packages" - pip install -U pip - pip install -r requirements.txt } # Install systemd services install_birdnetstream_services() { - cd BirdNET-stream - DIR=$(pwd) - GROUP=$USER + GROUP=birdnet debug "Setting up BirdNET stream systemd services" services="birdnet_recording.service birdnet_analyzis.service birdnet_miner.timer birdnet_miner.service birdnet_plotter.service birdnet_plotter.timer" - read -r -a services_array <<<"$services" - - for service in ${services_array[@]}; do - sudo cp daemon/systemd/templates/$service /etc/systemd/system/ + read -r -a services_array <<< "$services" + for service in "${services_array[@]}"; do + sudo cp "daemon/systemd/templates/$service" "/etc/systemd/system/" variables="DIR USER GROUP" for variable in $variables; do - sudo sed -i "s|<$variable>|${!variable}|g" /etc/systemd/system/$service + sudo sed -i "s|<$variable>|${!variable}|g" "/etc/systemd/system/$service" done done sudo systemctl daemon-reload @@ -90,15 +91,17 @@ install_php8() { } install_composer() { + cd /tmp php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"\nphp -r "if (hash_file('sha384', 'composer-setup.php') === '55ce33d7678c5a611085589f1f3ddf8b3c52d662cd01d4ba75c0ee0459970c2200a51f492d557530c71c15d8dba01eae') { echo 'Installer verified'; } else { echo 'Installer corrupt'; unlink('composer-setup.php'); } echo PHP_EOL;"\nphp composer-setup.php\nphp -r "unlink('composer-setup.php');" - sudo mv /composer.phar /usr/local/bin/composer + sudo mv ./composer.phar /usr/local/bin/composer + cd - } install_nodejs() { # Install nodejs curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash - export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")" - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm + export NVM_DIR="$([[ -z "${XDG_CONFIG_HOME-}" ]] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")" + [[ -s "$NVM_DIR/nvm.sh" ]] && \. "$NVM_DIR/nvm.sh" # This loads nvm nvm install 16 nvm use 16 install_requirements "npm" @@ -116,30 +119,60 @@ install_web_interface() { # Install nodejs 16 install_nodejs # Install Symfony web app - cd BirdNET-stream + cd "$WORKDIR" cd www debug "Creating nginx configuration" - cp nginx.conf /etc/nginx/sites-available/birdnet-stream.conf + sudo cp nginx.conf /etc/nginx/sites-available/birdnet-stream.conf sudo mkdir /var/log/nginx/birdnet/ - echo "Info: Please edit /etc/nginx/sites-available/birdnet-stream.conf to set the correct server name and paths" - sudo ln -s /etc/nginx/sites-available/birdnet-stream.conf /etc/nginx/sites-enabled/birdnet-stream.conf + sudo ln -s /etc/nginx/sites-available/birdnet-stream.conf /etc/nginx/sites-available/birdnet-stream.conf + debug "Info: Please edit /etc/nginx/sites-available/birdnet-stream.conf to set the correct server name and paths" sudo systemctl enable --now nginx sudo systemctl restart nginx debug "Retrieving composer dependencies" composer install + debug "PHP dependencies installed" debug "Installing nodejs dependencies" yarn install + debug "npm dependencies installed" debug "Building assets" yarn build + debug "Webpack assets built" debug "Web interface is available" debug "Please restart nginx after double check of /etc/nginx/sites-available/birdnet-stream.conf" } +change_value() { + local variable_name + variable_name="$1" + local variable_new_value + variable_new_value="$2" + local variable_filepath="$3" + sed -i "s|$variable_name=.*|$variable_name=\"$variable_new_value\"|g" "$variable_filepath" +} + +install_config() { + debug "Updating config" + cd "$WORKDIR" + cp ./config/birdnet.conf.example ./config/birdnet.conf + config_filepath="$WORKDIR/config/birdnet.conf" + change_value "DIR" "$WORKDIR" "$config_filepath" + change_value "PYTHON_VENV" "$PYTHON_VENV" "$config_filepath" + change_value "AUDIO_RECORDING" "true" "$config_filepath" + source "$config_filepath" + cd www + debug "Setup webapp .env" + cp .env.local.example .env.local + change_value "RECORDS_DIR" "$CHUNKS_FOLDER" ".env.local" +} + main() { - install_requirements $REQUIREMENTS + install_requirements "$REQUIREMENTS" install_birdnetstream install_birdnetstream_services install_web_interface + install_config + update_permissions "$WORKDIR" + debug "Installation done" } main diff --git a/utils/fix_permissions.sh b/utils/fix_permissions.sh index 514cfca..e6257c2 100644 --- a/utils/fix_permissions.sh +++ b/utils/fix_permissions.sh @@ -1,15 +1,13 @@ #! /usr/bin/env bash - +# Fix permissions on BirdNET-stream files when messed up set -e -DEBUG=${DEBUG:-1} +DEBUG=${DEBUG:-0} debug() { - if [ $DEBUG -eq 1 ]; then - echo "$1" - fi + [ $DEBUG -eq 1 ] && echo "$@" } -config_filepath="./config/analyzer.conf" +config_filepath="./config/birdnet.conf" if [ -f "$config_filepath" ]; then source "$config_filepath" @@ -18,7 +16,6 @@ else exit 1 fi - GROUP=birdnet sudo chown -R $USER:$GROUP $CHUNK_FOLDER diff --git a/www/package-lock.json b/www/package-lock.json index bf35891..6636154 100644 --- a/www/package-lock.json +++ b/www/package-lock.json @@ -16,6 +16,7 @@ "core-js": "^3.23.0", "git-revision-webpack-plugin": "^5.0.0", "regenerator-runtime": "^0.13.9", + "webpack": "^5.74.0", "webpack-notifier": "^1.15.0" } }, @@ -7630,7 +7631,6 @@ "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.74.0.tgz", "integrity": "sha512-A2InDwnhhGN4LYctJj6M1JEaGL7Luj6LOmyBHjcI8529cm5p6VXiTIW2sn6ffvEAKmveLzvu4jrihwXtPojlAA==", "dev": true, - "license": "MIT", "dependencies": { "@types/eslint-scope": "^3.7.3", "@types/estree": "^0.0.51", diff --git a/www/package.json b/www/package.json index 78e89b3..c65aa23 100644 --- a/www/package.json +++ b/www/package.json @@ -6,6 +6,7 @@ "core-js": "^3.23.0", "git-revision-webpack-plugin": "^5.0.0", "regenerator-runtime": "^0.13.9", + "webpack": "^5.74.0", "webpack-notifier": "^1.15.0" }, "license": "UNLICENSED", diff --git a/www/src/Controller/DisksController.php b/www/src/Controller/DisksController.php index acc5ed9..b38994e 100644 --- a/www/src/Controller/DisksController.php +++ b/www/src/Controller/DisksController.php @@ -10,8 +10,8 @@ class DisksController extends AbstractController { /** - * @Route("/disks/", name="disks_index") - * @Route("{_locale}/disks/", name="disks_index_i18n") + * @Route("/disks/", name="disks") + * @Route("{_locale}/disks/", name="disks_i18n") */ public function index() { return $this->render('disks/index.html.twig', [ diff --git a/www/templates/disks/index.html.twig b/www/templates/disks/index.html.twig index 33e6ddd..88bf108 100644 --- a/www/templates/disks/index.html.twig +++ b/www/templates/disks/index.html.twig @@ -1,3 +1,4 @@ +{% extends "base.html.twig" %} {% block content %}

{{ "Disk usage"|trans }}

diff --git a/www/templates/menu.html.twig b/www/templates/menu.html.twig index 375d41a..2c5b162 100644 --- a/www/templates/menu.html.twig +++ b/www/templates/menu.html.twig @@ -38,14 +38,23 @@ } %} - {% include 'utils/nav-item.html.twig' with { - route: 'logs', - text: 'View Logs'|trans - } %} - {% include 'utils/nav-item.html.twig' with { - route: 'services_status', - text: 'Status'|trans - } %} +
  • + {{ "Tools"|trans }} +
      + {% include 'utils/nav-item.html.twig' with { + route: 'logs', + text: 'View Logs'|trans + } %} + {% include 'utils/nav-item.html.twig' with { + route: 'services_status', + text: 'Status'|trans + } %} + {% include "utils/nav-item.html.twig" with { + route: 'disks', + text: 'Disks'|trans + } %} +
    +
    • diff --git a/www/translations/messages+intl-icu.fr.xlf b/www/translations/messages+intl-icu.fr.xlf index c28531a..0d6ea7c 100644 --- a/www/translations/messages+intl-icu.fr.xlf +++ b/www/translations/messages+intl-icu.fr.xlf @@ -11,7 +11,7 @@ BirdNET-stream is a realtime soundscape analyzis software powered by BirdNET AI. - BirdNET-stream est un logiciel d'analyse en temps réel de l'environement sonore basé sur BirdNET. + BirdNET-stream est un logiciel d'analyse en temps réel de l'environnement sonore basé sur BirdNET. It aims to be able to run on any computer with a microphone. diff --git a/www/yarn.lock b/www/yarn.lock index 461237e..4a34fbd 100644 --- a/www/yarn.lock +++ b/www/yarn.lock @@ -4252,7 +4252,7 @@ "resolved" "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.2.3.tgz" "version" "3.2.3" -"webpack@^2.0.0 || ^3.0.0 || ^4.0.0 || ^5.0.0", "webpack@^4.0.0 || ^5.0.0", "webpack@^4.37.0 || ^5.0.0", "webpack@^5.0.0", "webpack@^5.1.0", "webpack@^5.72", "webpack@>=2", "webpack@>=4.0.0 <6.0.0", "webpack@>=5.0.0", "webpack@4.x.x || 5.x.x": +"webpack@^2.0.0 || ^3.0.0 || ^4.0.0 || ^5.0.0", "webpack@^4.0.0 || ^5.0.0", "webpack@^4.37.0 || ^5.0.0", "webpack@^5.0.0", "webpack@^5.1.0", "webpack@^5.72", "webpack@^5.74.0", "webpack@>=2", "webpack@>=4.0.0 <6.0.0", "webpack@>=5.0.0", "webpack@4.x.x || 5.x.x": "integrity" "sha512-A2InDwnhhGN4LYctJj6M1JEaGL7Luj6LOmyBHjcI8529cm5p6VXiTIW2sn6ffvEAKmveLzvu4jrihwXtPojlAA==" "resolved" "https://registry.npmjs.org/webpack/-/webpack-5.74.0.tgz" "version" "5.74.0"