From 4f09a2dd4ea26fd1f28299036c2003010c32afe0 Mon Sep 17 00:00:00 2001 From: Samuel ORTION Date: Wed, 24 Aug 2022 11:58:15 +0200 Subject: [PATCH] daemon: Remove birdnet_miner and call birdnet_output_to_sql at each new model execution --- CHANGELOG.md | 5 +- INSTALL.md | 4 +- daemon/birdnet_analyzis.sh | 23 ++- ...dnet_miner.sh => birdnet_output_to_sql.sh} | 33 +--- daemon/plotter/chart.py | 165 ++++++++++-------- .../systemd/templates/birdnet_miner.service | 13 -- daemon/systemd/templates/birdnet_miner.timer | 9 - docs/DOCKER.md | 9 +- install.sh | 6 +- uninstall.sh | 9 + www/src/Controller/HomeController.php | 34 +++- www/templates/stats.html.twig | 126 +++++++------ 12 files changed, 248 insertions(+), 188 deletions(-) rename daemon/{birdnet_miner.sh => birdnet_output_to_sql.sh} (82%) delete mode 100644 daemon/systemd/templates/birdnet_miner.service delete mode 100644 daemon/systemd/templates/birdnet_miner.timer diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c82771..5dff610 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,9 @@ - Add docker compose port - Improve install script +- Add base uninstall script (need deeper work) - Add ttyd for systemd logging -## v0.0.1-rc +## v0.0.1-rc (2022-08-18) - Integrate BirdNET-Analyzer as submodule - Add birdnet_recording service @@ -15,4 +16,4 @@ - Add /today/species and /today/{date}/species/{id} endpoints - Add records deletion button and /records/delete endpoint as well as bulk deletion (select all button on /today/species/{id} endpoint) - Add systemd status page /status -- Add i18n for webapp (not species name), en|fr only for the moment +- Add i18n for webapp (not species name), en|fr only for the moment \ No newline at end of file diff --git a/INSTALL.md b/INSTALL.md index 0b3a110..9ebb0fe 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -129,7 +129,7 @@ sudo mv /composer.phar /usr/local/bin/composer ```bash cd www -composer install +composer install --no-dev --prefer-dist --optimize-autoloader ``` ### Install nodejs and npm @@ -147,7 +147,7 @@ nvm use 16 ``` ```bash -sudo dnf install npm +sudo apt-get install npm ``` ```bash diff --git a/daemon/birdnet_analyzis.sh b/daemon/birdnet_analyzis.sh index 8943faf..54d01de 100755 --- a/daemon/birdnet_analyzis.sh +++ b/daemon/birdnet_analyzis.sh @@ -3,9 +3,7 @@ set -e DEBUG=${DEBUG:-1} debug() { - if [ $DEBUG -eq 1 ]; then - echo "$1" - fi + [[ $DEBUG -eq 1 ]] && echo "$@" } config_filepath="./config/birdnet.conf" @@ -64,7 +62,9 @@ check_prerequisites() { # Get array of audio chunks to be processed get_chunk_list() { - find "${CHUNK_FOLDER}/in" -type f -name '*.wav' -exec basename {} \; ! -size 0 | sort + chunk_list=($(ls ${CHUNK_FOLDER}/in)) + echo "${chunk_list}" + # find "${CHUNK_FOLDER}/in" -type f -name '*.wav' -exec basename {} \; ! -size 0 | sort } # Perform audio chunk analysis on one chunk @@ -75,13 +75,22 @@ analyze_chunk() { mkdir -p "$output_dir" date=$(echo $chunk_name | cut -d'_' -f2) week=$(./daemon/weekof.sh $date) - $PYTHON_EXECUTABLE ./analyzer/analyze.py --i $chunk_path --o "$output_dir/model.out.csv" --lat $LATITUDE --lon $LONGITUDE --week $week --min_conf $CONFIDENCE --threads 4 --rtype csv + if [[ ! -z "${THREADS}" ]]; then + threads="--threads ${THREADS}" + else + threads="" + fi + $PYTHON_EXECUTABLE ./analyzer/analyze.py --i $chunk_path --o "$output_dir/model.out.csv" --lat $LATITUDE --lon $LONGITUDE --week $week --min_conf $CONFIDENCE $threads --rtype csv debug "Model output written to $output_dir/model.out.csv" + bash ./daemon/birdnet_output_to_sql.sh "$output_dir/model.out.csv" + debug "Dumped to SQL database" } # Perform audio chunk analysis on all recorded chunks analyze_chunks() { - for chunk_name in $(get_chunk_list); do + local chunks + chunks="${1}" + for chunk_name in "${chunks}"; do if [[ -f "${CHUNK_FOLDER}/out/$chunk_name.d/model.out.csv" ]]; then debug "Skipping $chunk_name, as it has already been analyzed" else @@ -98,4 +107,4 @@ check_prerequisites chunks=$(get_chunk_list) # Analyze all chunks in working directory -analyze_chunks $chunks +analyze_chunks "$chunks" \ No newline at end of file diff --git a/daemon/birdnet_miner.sh b/daemon/birdnet_output_to_sql.sh similarity index 82% rename from daemon/birdnet_miner.sh rename to daemon/birdnet_output_to_sql.sh index d03075d..2771f97 100755 --- a/daemon/birdnet_miner.sh +++ b/daemon/birdnet_output_to_sql.sh @@ -1,15 +1,13 @@ #! /usr/bin/env bash -# Extract observations from a model output folder +# Extract observations from a model output file into SQL database # DEBUG=${DEBUG:-1} set -e # set -x - +DEBUG=${DEBUG:-1} debug() { - if [ $DEBUG -eq 1 ]; then - echo "$1" - fi + [[ $DEBUG -eq 1 ]] && echo "$@" } # Load bash library to deal with BirdNET-stream database @@ -18,16 +16,6 @@ source ./daemon/database/scripts/database.sh # Load config source ./config/birdnet.conf # Check config -if [[ -z ${CHUNK_FOLDER} ]]; then - echo "CHUNK_FOLDER is not set" - exit 1 -else - if [[ ! -d ${CHUNK_FOLDER}/out ]]; then - echo "CHUNK_FOLDER does not exist: ${CHUNK_FOLDER}/out" - echo "Cannot extract observations." - exit 1 - fi -fi if [[ -z ${LATITUDE} ]]; then echo "LATITUDE is not set" @@ -39,10 +27,6 @@ if [[ -z ${LONGITUDE} ]]; then exit 1 fi -model_outputs() { - ls ${CHUNK_FOLDER}/out/*/model.out.csv -} - source_wav() { model_output_path=$1 model_output_dir=$(dirname $model_output_path) @@ -107,13 +91,6 @@ save_observations() { done } -main() { - # # Remove all junk observations - # ./daemon/birdnet_clean.sh - # Get model outputs - for model_output in $(model_outputs); do - save_observations $model_output - done -} +model_output_path="$1" -main +save_observations $model_output_path \ No newline at end of file diff --git a/daemon/plotter/chart.py b/daemon/plotter/chart.py index 5028dfe..ca2b930 100755 --- a/daemon/plotter/chart.py +++ b/daemon/plotter/chart.py @@ -6,90 +6,119 @@ import matplotlib.pyplot as plt from matplotlib.colors import LogNorm import seaborn as sns from datetime import datetime +import os +import glob CONFIG = { "readings": 10, "palette": "Greens", "db": "./var/db.sqlite", - "date": datetime.now().strftime("%Y-%m-%d") - # "date": "2022-08-15" + "date": datetime.now().strftime("%Y-%m-%d"), + "charts_dir": "./var/charts" } -db = sqlite3.connect(CONFIG['db']) +db = None -df = pd.read_sql_query("""SELECT common_name, date, location_id, confidence - FROM observation - INNER JOIN taxon - ON observation.taxon_id = taxon.taxon_id""", db) -df['date'] = pd.to_datetime(df['date']) -df['hour'] = df['date'].dt.hour -df['date'] = df['date'].dt.date -df['date'] = df['date'].astype(str) -df_on_date = df[df['date'] == CONFIG['date']] - -top_on_date = (df_on_date['common_name'].value_counts()[:CONFIG['readings']]) -if top_on_date.empty: - print("No observations on {}".format(CONFIG['date'])) - exit() +def get_database(): + global db + if db is None: + db = sqlite3.connect(CONFIG["db"]) + return db -df_top_on_date = df_on_date[df_on_date['common_name'].isin(top_on_date.index)] +def chart(date): + db = get_database() + df = pd.read_sql_query(f"""SELECT common_name, date, location_id, confidence + FROM observation + INNER JOIN taxon + ON observation.taxon_id = taxon.taxon_id + WHERE STRFTIME("%Y-%m-%d", `date`) = '{date}'""", db) + df['date'] = pd.to_datetime(df['date']) + df['hour'] = df['date'].dt.hour + df['date'] = df['date'].dt.date + df['date'] = df['date'].astype(str) + df_on_date = df[df['date'] == date] + + top_on_date = (df_on_date['common_name'].value_counts()[:CONFIG['readings']]) + if top_on_date.empty: + print("No observations on {}".format(date)) + return + else: + print(f"Found observations on {date}") + + df_top_on_date = df_on_date[df_on_date['common_name'].isin(top_on_date.index)] -# Create a figure with 2 subplots -fig, axs = plt.subplots(1, 2, figsize=(20, 5), gridspec_kw=dict( - width_ratios=[2, 6])) -plt.subplots_adjust(left=None, bottom=None, right=None, - top=None, wspace=0, hspace=0) + # Create a figure with 2 subplots + fig, axs = plt.subplots(1, 2, figsize=(20, 5), gridspec_kw=dict( + width_ratios=[2, 6])) + plt.subplots_adjust(left=None, bottom=None, right=None, + top=None, wspace=0, hspace=0) -# Get species frequencies -frequencies_order = pd.value_counts(df_top_on_date['common_name']).iloc[:CONFIG['readings']].index -# Get min max confidences -confidence_minmax = df_top_on_date.groupby('common_name')['confidence'].max() -confidence_minmax = confidence_minmax.reindex(frequencies_order) -# Norm values for color palette -norm = plt.Normalize(confidence_minmax.values.min(), - confidence_minmax.values.max()) + # Get species frequencies + frequencies_order = pd.value_counts(df_top_on_date['common_name']).iloc[:CONFIG['readings']].index + # Get min max confidences + confidence_minmax = df_top_on_date.groupby('common_name')['confidence'].max() + confidence_minmax = confidence_minmax.reindex(frequencies_order) + # Norm values for color palette + norm = plt.Normalize(confidence_minmax.values.min(), + confidence_minmax.values.max()) -colors = plt.cm.Greens(norm(confidence_minmax)) -plot = sns.countplot(y='common_name', data=df_top_on_date, palette=colors, order=frequencies_order, ax=axs[0]) + colors = plt.cm.Greens(norm(confidence_minmax)) + plot = sns.countplot(y='common_name', data=df_top_on_date, palette=colors, order=frequencies_order, ax=axs[0]) -plot.set(ylabel=None) -plot.set(xlabel="Detections") + plot.set(ylabel=None) + plot.set(xlabel="Detections") -heat = pd.crosstab(df_top_on_date['common_name'], df_top_on_date['hour']) -# Order heatmap Birds by frequency of occurrance -heat.index = pd.CategoricalIndex(heat.index, categories=frequencies_order) -heat.sort_index(level=0, inplace=True) + heat = pd.crosstab(df_top_on_date['common_name'], df_top_on_date['hour']) + # Order heatmap Birds by frequency of occurrance + heat.index = pd.CategoricalIndex(heat.index, categories=frequencies_order) + heat.sort_index(level=0, inplace=True) -hours_in_day = pd.Series(data=range(0, 24)) -heat_frame = pd.DataFrame(data=0, index=heat.index, columns=hours_in_day) -heat = (heat + heat_frame).fillna(0) + hours_in_day = pd.Series(data=range(0, 24)) + heat_frame = pd.DataFrame(data=0, index=heat.index, columns=hours_in_day) + heat = (heat + heat_frame).fillna(0) -# Generate heatmap plot -plot = sns.heatmap( - heat, - norm=LogNorm(), - annot=True, - annot_kws={ - "fontsize": 7 - }, - fmt="g", - cmap=CONFIG['palette'], - square=False, - cbar=False, - linewidth=0.5, - linecolor="Grey", - ax=axs[1], - yticklabels=False) -plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7) + # Generate heatmap plot + plot = sns.heatmap( + heat, + norm=LogNorm(), + annot=True, + annot_kws={ + "fontsize": 7 + }, + fmt="g", + cmap=CONFIG['palette'], + square=False, + cbar=False, + linewidth=0.5, + linecolor="Grey", + ax=axs[1], + yticklabels=False) + plot.set_xticklabels(plot.get_xticklabels(), rotation=0, size=7) -for _, spine in plot.spines.items(): - spine.set_visible(True) + for _, spine in plot.spines.items(): + spine.set_visible(True) -plot.set(ylabel=None) -plot.set(xlabel="Hour of day") -plt.suptitle(f"Top {CONFIG['readings']} species on {CONFIG['date']}", fontsize=14) -plt.text(15, 11, f"(Updated on {datetime.now().strftime('%Y/%m-%d %H:%M')})") -plt.savefig(f"./var/charts/chart_{CONFIG['date']}.png", dpi=300) -plt.close() + plot.set(ylabel=None) + plot.set(xlabel="Hour of day") + plt.suptitle(f"Top {CONFIG['readings']} species on {CONFIG['date']}", fontsize=14) + plt.text(15, 11, f"(Updated on {datetime.now().strftime('%Y/%m-%d %H:%M')})") + plt.savefig(f"./var/charts/chart_{CONFIG['date']}.png", dpi=300) + print(f"Plot for {date} saved.") + plt.close() -db.close() \ No newline at end of file +def main(): + done_charts = glob.glob(f"{CONFIG['charts_dir']}/*.png") + last_modified = max(done_charts, key=os.path.getctime) + last_modified_date = last_modified.split("_")[-1].split(".")[0] + missing_dates = pd.date_range(start=last_modified_date, end=CONFIG['date'], freq='D') + print(missing_dates) + for missing_date in missing_dates: + date = missing_date.strftime("%Y-%m-%d") + chart(date) + chart(CONFIG['date']) + if db is not None: + db.close() + print("Done.") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/daemon/systemd/templates/birdnet_miner.service b/daemon/systemd/templates/birdnet_miner.service deleted file mode 100644 index d24c1f8..0000000 --- a/daemon/systemd/templates/birdnet_miner.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=BirdNET-stream miner service - -[Service] -Type=simple -User= -Group= -WorkingDirectory= -ExecStart=bash ./daemon/birdnet_miner.sh -RemainAfterExit=yes - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/daemon/systemd/templates/birdnet_miner.timer b/daemon/systemd/templates/birdnet_miner.timer deleted file mode 100644 index 931c73f..0000000 --- a/daemon/systemd/templates/birdnet_miner.timer +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=BirdNET-stream miner Timer - -[Timer] -OnCalendar=*:0/15 -Unit=birdnet_miner.service - -[Install] -WantedBy=timers.target \ No newline at end of file diff --git a/docs/DOCKER.md b/docs/DOCKER.md index 94fd560..f42a61d 100644 --- a/docs/DOCKER.md +++ b/docs/DOCKER.md @@ -34,10 +34,13 @@ Then, create your dotenv file and populate it with your own configuration (for i cp .env.example .env ``` -Then, run docker-compose: +You may need to adapt the listening ports of the services or other configuration parameters. +In general all variables stated with ${VARIABLE:-default} inside [../docker-compose.yml](../docker-compose.yml) can be override in the .env file using `VARIABLE=value`. + +Once that is done, you can build and start docker services: ```bash -# Build image (first time only) +# Build images (first time only) docker compose build # Run docker compose up # add `-d`, to run in background @@ -48,4 +51,4 @@ docker compose down For a one liner: ```bash docker compose up --build -``` +``` \ No newline at end of file diff --git a/install.sh b/install.sh index 4d5c72a..770b32c 100755 --- a/install.sh +++ b/install.sh @@ -67,7 +67,7 @@ install_birdnetstream_services() { DIR="$WORKDIR" cd "$WORKDIR" debug "Setting up BirdNET stream systemd services" - services="birdnet_recording.service birdnet_analyzis.service birdnet_miner.timer birdnet_miner.service birdnet_plotter.service birdnet_plotter.timer" + services="birdnet_recording.service birdnet_analyzis.service birdnet_plotter.service birdnet_plotter.timer" read -r -a services_array <<<"$services" for service in ${services_array[@]}; do sudo cp "daemon/systemd/templates/$service" "/etc/systemd/system/" @@ -78,7 +78,7 @@ install_birdnetstream_services() { done sudo sed -i "s||$WORKDIR/$PYTHON_VENV|g" "/etc/systemd/system/birdnet_plotter.service" sudo systemctl daemon-reload - enabled_services="birdnet_recording.service birdnet_analyzis.service birdnet_miner.timer birdnet_plotter.timer" + enabled_services="birdnet_recording.service birdnet_analyzis.service birdnet_plotter.timer" read -r -a services_array <<<"$services" for service in ${services_array[@]}; do debug "Enabling $service" @@ -163,7 +163,7 @@ setup_http_server() { fi debug "Enable birdnet.lan domain" sudo ln -s /etc/nginx/sites-available/birdnet-stream.conf /etc/nginx/sites-enabled/birdnet-stream.conf - debug "Info: Please edit /etc/nginx/sites-available/birdnet-stream.conf to set the correct server name and paths" + debug "INFO: Please edit /etc/nginx/sites-available/birdnet-stream.conf to set the correct server name and paths" debug "Setup nginx variables the best way possible" sudo sed -i "s||$WORKDIR/www/public/|g" /etc/nginx/sites-available/birdnet-stream.conf sudo sed -i "s||$CHUNK_FOLDER/out|g" /etc/nginx/sites-available/birdnet-stream.conf diff --git a/uninstall.sh b/uninstall.sh index 31df398..4baaf96 100644 --- a/uninstall.sh +++ b/uninstall.sh @@ -38,3 +38,12 @@ uninstall_webapp() { sudo unlink /etc/nginx/sites-enabled/birdnet-stream.conf sudo systemctl restart nginx } + +main() { + echo "WARNING: This will remove all BirdNET-stream related files and services. \ + Note that it may forget some special configuration." + uninstall_webapp + uninstall_birdnet_services +} + +main \ No newline at end of file diff --git a/www/src/Controller/HomeController.php b/www/src/Controller/HomeController.php index 19248d1..3b2a86f 100644 --- a/www/src/Controller/HomeController.php +++ b/www/src/Controller/HomeController.php @@ -25,11 +25,15 @@ class HomeController extends AbstractController * @Route("", name="home") * @Route("/{_locale<%app.supported_locales%>}/", name="home_i18n") */ - public function index() + public function index(Request $request) { + $date = $request->get("on"); + if ($date == null) { + $date = date("Y-m-d"); + } return $this->render('index.html.twig', [ - "stats" => $this->get_stats(), - "charts" => $this->last_chart_generated(), + "stats" => $this->get_stats($date), + "charts" => $this->last_chart_generated($date), ]); } @@ -42,11 +46,12 @@ class HomeController extends AbstractController return $this->render('about/index.html.twig', []); } - private function get_stats() + private function get_stats($date) { $stats = array(); $stats["most-recorded-species"] = $this->get_most_recorded_species(); $stats["last-detected-species"] = $this->get_last_recorded_species(); + $stats["number-of-species-detected"] = $this->get_number_of_species_detected($date); return $stats; } @@ -86,6 +91,27 @@ class HomeController extends AbstractController return $species; } + private function get_number_of_species_detected($date) + { + $count = 0; + $sql = "SELECT COUNT(`taxon_id`) AS contact_count + FROM `observation` + WHERE STRFTIME('%Y-%m-%d', `date`) = :date + GROUP BY `taxon_id`"; + try { + $stmt = $this->connection->prepare($sql); + $stmt->bindValue(":date", $date); + $result = $stmt->executeQuery(); + $output = $result->fetchAllAssociative(); + if ($output != null) { + $count = $output[0]["contact_count"]; + } + } catch (\Exception $e) { + $this->logger->error($e->getMessage()); + } + return $count; + } + private function last_chart_generated() { $files = glob($this->getParameter('kernel.project_dir') . '/../var/charts/*.png'); diff --git a/www/templates/stats.html.twig b/www/templates/stats.html.twig index b1e35a0..493a151 100644 --- a/www/templates/stats.html.twig +++ b/www/templates/stats.html.twig @@ -1,50 +1,78 @@
-

{{ "Quick Stats" | trans }}

-
    -
  • - {{ "Most recorded species" | trans }}: - {% if stats["most-recorded-species"] is defined and stats["most-recorded-species"]|length > 0 %} - - {{ stats["most-recorded-species"]["scientific_name"] }} - - ({{ stats["most-recorded-species"]["common_name"] }}) - {{ "with" | trans }} - - {{ stats["most-recorded-species"]["contact_count"] }} - - {{ "contacts" | trans }}. - {% else %} - {{ "No species in database." | trans }} - {% endif %} -
    • -
  • - {{ "Last detected species" | trans }}: - {% if stats["last-detected-species"] is defined and stats["last-detected-species"]|length > 0 %} - - {{ stats["last-detected-species"]["scientific_name"] }} - - ({{ stats["last-detected-species"]["common_name"] }}) - {{ "with" | trans }} - - {{ stats["last-detected-species"]["confidence"] }} - - {{ "AI confidence" | trans }} - - {% set date = stats["last-detected-species"]["date"] %} - {% if date | date("Y-m-d") == "now" | date("Y-m-d") %} - {{ "today" | trans }} - {% else %} - {{ "on" | trans }} - {{ date | format_datetime("full", "none") }} - {% endif %} - at - - {{ date | date("H:i") }} - - . - {% else %} - {{ "No species in database" | trans }} - {% endif %} -
    • -
-
\ No newline at end of file +

+ {{ 'Quick Stats'|trans }} +

+
    +
  • + {{ 'Most recorded species'|trans }}:{% if + stats['most-recorded-species'] is defined + and (stats['most-recorded-species']|length) > 0 %} + + {{ stats['most-recorded-species']['scientific_name'] }} + + ( + {{ stats['most-recorded-species']['common_name'] }} + ) + {{ 'with'|trans }} + + {{ stats['most-recorded-species']['contact_count'] }} + + {{ 'contacts'|trans }}. + {% else %} + {{ 'No species in database.'|trans }} + {% endif %} +
    • +
  • + {{ 'Last detected species'|trans }}:{% if + stats['last-detected-species'] is defined + and (stats['last-detected-species']|length) > 0 %} + + {{ stats['last-detected-species']['scientific_name'] }} + + ( + {{ stats['last-detected-species']['common_name'] }} + ) + {{ 'with'|trans }} + + {{ stats['last-detected-species']['confidence'] }} + + {{ 'AI confidence'|trans }} + + {% set date = stats['last-detected-species']['date'] %} + {% if (date|date('Y-m-d')) == ('now'|date('Y-m-d')) %} + {{ 'today'|trans }} + {% else %} + {{ 'on'|trans }} + {{ date|format_datetime('full', 'none') }} + {% endif %}at + {{ date|date('H:i') }} + . + {% else %} + {{ 'No species in database'|trans }} + {% endif %} +
    • +
  • + {% set today = 'now'|date('Y-m-d') %} + {% set date = app.request.get('on') %} + {% if + stats['number-of-species-detected'] is defined + and stats['number-of-species-detected'] > 0 %} + {% if today == date %} + {{ 'Number of species detected today: '|trans }} + {% else %} + {{ 'Number of species detected on '|trans }} + {{ date|format_datetime('full', 'none') }}: + {% endif %} + {{ stats['number-of-species-detected'] }}. + {% else %} + {# {{ 'No species detected today'|trans }} #} + {% if today == date %} + {{ 'No species detected today.'|trans }} + {% else %} + {{ 'No species detected on '|trans }} + {{ date|format_datetime('full', 'none') }} + {% endif %} + {% endif %} +
    • +
+