Merge pull request #98 from reynico/iss_support

Add ISS reception and decoding support

ISS.md

@@ -0,0 +1,32 @@
+![Raspberry NOAA](header_1600.png)
+
+This project now includes full reception and decoding of SSTV transmissions from the ISS.
+
+# ISS reception and scheduling
+Reception is quite easy: The ISS TX works on 145.800 Mhz, Narrow FM. So we just need to tune our RTLSDR receiver and that's all.
+
+Scheduling is governed by the `SCHEDULE_ISS` inside `.noaa.conf`. Setting it to `true` will schedule future ISS passes.
+
+# ISS decoding
+This is the difficult part: Decoding PD120 needs custom software. Right now the best Linux software for SSTV (in mu humble opinion) is [QSSTV](http://users.telenet.be/on4qz/qsstv/index.html). QSSTV is great, works with a lot of modes, has auto slant and signal detection. On the other hand, QSSTV only works in GUI mode as it's written in QT (hence Q-SSTV) and that's not situable for this project as we don't run any Window Server.
+
+Surfing the Internet I've found Martin Bernardi's (and team) [final work about a PD120 modulator - demodulator](https://github.com/martinber/rtlsdr_sstv) so I put hands on slightly modifying it to adjust the project to my needs.
+
+## Decoding behavior
+There are a few things to take care about
+
+1. A single ISS pass during a SSTV event could have zero to three SSTV transmissions (based on an average pass time of 10 minutes) so we need to find a way to detect the transmission header to fire up the decoder and store each image.
+
+![PD 120 header](pd120_header.png)
+
+2. A simple way to detect the header on an audio file (once digitized with scipy.io) is to sample the header as an absolute numeric representation, apply a threshold during certain amount of time. So if the header is present, mark the stream and do the same for each new header that ocurrs in about `this_timestamp + 120 seconds` so we don't get any false positives in the middle of the PD120 carrier.
+
+3. A Satellite transmission to the earth has doppler efect so the frequency changes over the time. This is a problem when you plot audio as data. So I had to include a high pass filter to remove the DC offset from the audio recorded by `rtl_fm`.
+
+# Upgrading procedure
+There's a migration script that handles the required modifications over the database as well as the web files. Take in mind that you need to have the web panel version of raspberry-noaa running.
+
+1. `cd /home/pi/raspberry-noaa`
+2. `git fetch -pt && git pull`
+3. `cd migrations/`
+4. `./20201292-iss.sh`

migrations/20201292-iss.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+set -e
+
+### Run as a normal user
+if [ $EUID -eq 0 ]; then
+    echo "This script shouldn't be run as root."
+    exit 1
+fi
+
+## import common lib
+. "$HOME/.noaa.conf"
+. "$NOAA_HOME/common.sh"
+
+if [ -f "$NOAA_HOME/demod.py" ]; then
+    log "Seems like you already have run this migration before" "ERROR"
+    exit 1
+fi
+
+if [ ! -f "$NOAA_HOME/panel.db" ]; then
+    log "Seems like there's no panel.db database in your project folder" "ERROR"
+    exit 1
+fi
+
+STEPS="6"
+
+datetime=$(date +"%Y%m%d-%H%M%S")
+
+log "1/$STEPS: Backing up database" "INFO"
+cp "$NOAA_HOME/panel.db" "$NOAA_HOME/panel.db.bak-$datetime"
+log "1/$STEPS: Database backup done: panel.db.bak-$datetime" "INFO"
+
+log "2/$STEPS: Creating new columns" "INFO"
+set +e
+sqlite3 "$NOAA_HOME/panel.db" "alter table decoded_passes add column img_count integer;"
+sqlite3 "$NOAA_HOME/panel.db" "alter table decoded_passes add column sat_type integer;"
+set -e
+log "2/$STEPS: img_count and sat_type columns created" "INFO"
+
+
+log "3/$STEPS: Migrating is_noaa column" "INFO"
+sqlite3 "$NOAA_HOME/panel.db" "update decoded_passes set sat_type = is_noaa;"
+log "3/$STEPS: is_noaa column migration done" "INFO"
+
+
+log "4/$STEPS: Setting up SCHEDULE_ISS on .noaa.conf" "INFO"
+set +e
+if ! grep -q SCHEDULE_ISS "$HOME/.noaa.conf"; then
+    echo "SCHEDULE_ISS=\"false\"" >> "$HOME/.noaa.conf"
+    log "4/$STEPS: SCHEDULE_ISS is set now on .noaa.conf" "INFO"
+else
+    log "4/$STEPS: SCHEDULE_ISS was already set on .noaa.conf" "INFO"
+fi
+set -e
+
+log "5/$STEPS: Updating PHP files" "INFO"
+sudo cp "$NOAA_HOME/templates/webpanel/Model/Conn.php" "/var/www/wx/Model/Conn.php"
+sudo cp "$NOAA_HOME/templates/webpanel/Views/V_viewLastImages.php" "/var/www/wx/Views/V_viewLastImages.php"
+log "5/$STEPS: PHP files updated" "INFO"
+
+
+log "6/$STEPS: Installing pd120_decoder" "INFO"
+if [ -f "$NOAA_HOME/demod.py" ]; then
+    log "6/$STEPS: pd120_decoder already installed" "INFO"
+else
+    wget -qr https://github.com/reynico/pd120_decoder/archive/master.zip -O /tmp/master.zip
+    (
+        cd /tmp
+        unzip master.zip
+        cd pd120_decoder-master/pd120_decoder/
+        pip3 install --user -r requirements.txt
+        cp "{demod.py,utils.py}" "$NOAA_HOME"
+    )
+    log "6/$STEPS: pd120_decoder installed" "INFO"
+fi

receive.sh

@@ -53,9 +53,9 @@ done
 rm "${NOAA_HOME}/map/${3}-map.png"
 
 if [ "${SUN_ELEV}" -gt "${SUN_MIN_ELEV}" ]; then
-	sqlite3 /home/pi/raspberry-noaa/panel.db "insert into decoded_passes (pass_start, file_path, daylight_pass, is_noaa) values ($5,\"$3\", 1,1);"
+	sqlite3 /home/pi/raspberry-noaa/panel.db "insert into decoded_passes (pass_start, file_path, daylight_pass, sat_type) values ($5,\"$3\", 1,1);"
 else
-	sqlite3 /home/pi/raspberry-noaa/panel.db "insert into decoded_passes (pass_start, file_path, daylight_pass, is_noaa) values ($5,\"$3\", 0,1);"
+	sqlite3 /home/pi/raspberry-noaa/panel.db "insert into decoded_passes (pass_start, file_path, daylight_pass, sat_type) values ($5,\"$3\", 0,1);"
 fi
 
 pass_id=$(sqlite3 /home/pi/raspberry-noaa/panel.db "select id from decoded_passes order by id desc limit 1;")

receive_iss.sh

@@ -1,3 +1,56 @@
-#!/bin/sh
-datetime=$(date +"%Y%m%d-%H%M%S")
-timeout 660 /usr/local/bin/rtl_fm -M fm -f 145.8M -s 48k -g $GAIN -p 55 -E wav -E deemp -F 9 - | /usr/bin/sox -t raw -e signed -c 1 -b 16 -r 48000 - /usr/share/html/iss/iss-$datetime.wav rate 11025
+#!/bin/bash
+
+### Run as a normal user
+if [ $EUID -eq 0 ]; then
+    echo "This script shouldn't be run as root."
+    exit 1
+fi
+
+## import common lib
+. "$HOME/.noaa.conf"
+. "$HOME/.tweepy.conf"
+. "$NOAA_HOME/common.sh"
+
+if pgrep "rtl_fm" > /dev/null
+then
+    log "There is an already running rtl_fm instance but I dont care for now, I prefer this pass" "INFO"
+    pkill -9 -f rtl_fm
+fi
+
+# $1 = Satellite Name
+# $2 = Frequency
+# $3 = FileName base
+# $4 = TLE File
+# $5 = EPOC start time
+# $6 = Time to capture
+# $7 = Satellite max elevation
+
+log "Starting rtl_fm record" "INFO"
+timeout "${6}" /usr/local/bin/rtl_fm ${BIAS_TEE} -M fm -f 145.8M -s 48k -g $GAIN -E dc -E wav -E deemp -F 9 - | sox -t raw -r 48k -c 1 -b 16 -e s - -t wav "${NOAA_OUTPUT}/audio/${3}.wav" rate 11025
+
+if [ -f "$NOAA_HOME/demod.py" ]; then
+    log "Decoding ISS pass" "INFO"
+    python3 "$NOAA_HOME/demod.py" "${NOAA_OUTPUT}/audio/${3}.wav" "${NOAA_OUTPUT}/images/"
+    decoded_pictures="$(find ${NOAA_OUTPUT}/images/ -iname "${3}*png")"
+    img_count=0
+    for image in $decoded_pictures; do
+        log "Decoded image: $image" "INFO"
+        ((img_count++))
+    done
+
+    if [ "$img_count" -gt 0 ]; then
+        /usr/bin/convert -thumbnail 300 "${NOAA_OUTPUT}/images/${3}-0.png" "${NOAA_OUTPUT}/images/thumb/${3}-0.png"
+        sqlite3 "$NOAA_HOME/panel.db" "insert into decoded_passes (pass_start, file_path, daylight_pass, sat_type, img_count) values ($5,\"$3\",1,2,$img_count);"
+        pass_id=$(sqlite3 "$NOAA_HOME/panel.db" "select id from decoded_passes order by id desc limit 1;")
+        if [ -n "$CONSUMER_KEY" ]; then
+            log "Posting to Twitter" "INFO"
+            if [ "$img_count" -eq 1 ]; then
+                python3 "${NOAA_HOME}/post.py" "$1 ${START_DATE} Resolución completa: https://weather.reyni.co/detail.php?id=$pass_id" "$7" "${NOAA_OUTPUT}/images/${3}-0.png"
+            elif [ "$img_count" -eq 2 ]; then
+                /usr/bin/convert -thumbnail 300 "${NOAA_OUTPUT}/images/${3}-1.png" "${NOAA_OUTPUT}/images/thumb/${3}-1.png"
+                python3 "${NOAA_HOME}/post.py" "$1 ${START_DATE} Mas imagenes: https://weather.reyni.co/detail.php?id=$pass_id" "$7" "${NOAA_OUTPUT}/images/${3}-0.png" "${NOAA_OUTPUT}/images/${3}-1.png"
+            fi
+        fi
+        sqlite3 "$NOAA_HOME/panel.db" "update predict_passes set is_active = 0 where (predict_passes.pass_start) in (select predict_passes.pass_start from predict_passes inner join decoded_passes on predict_passes.pass_start = decoded_passes.pass_start where decoded_passes.id = $pass_id);"
+    fi
+fi

receive_meteor.sh

@@ -82,7 +82,7 @@ if [ -f "${METEOR_OUTPUT}/${3}.dec" ]; then
     rm "${METEOR_OUTPUT}/${3}.bmp"
     rm "${METEOR_OUTPUT}/${3}.dec"
 
-    sqlite3 /home/pi/raspberry-noaa/panel.db "insert into decoded_passes (pass_start, file_path, daylight_pass, is_noaa) values ($5,\"$3\", 1,0);"
+    sqlite3 /home/pi/raspberry-noaa/panel.db "insert into decoded_passes (pass_start, file_path, daylight_pass, sat_type) values ($5,\"$3\", 1,0);"
     pass_id=$(sqlite3 /home/pi/raspberry-noaa/panel.db "select id from decoded_passes order by id desc limit 1;")
     if [ -n "$CONSUMER_KEY" ]; then
         log "Posting to Twitter" "INFO"

schedule.sh

@@ -10,24 +10,23 @@ fi
 . "$HOME/.noaa.conf"
 . "$NOAA_HOME/common.sh"
 
-### Run as a normal user
-if [ $EUID -eq 0 ]; then
-    die "This script shouldn't be run as root."
-fi
-
 wget -qr http://www.celestrak.com/NORAD/elements/weather.txt -O "${NOAA_HOME}"/predict/weather.txt
 wget -qr http://www.celestrak.com/NORAD/elements/amateur.txt -O "${NOAA_HOME}"/predict/amateur.txt
 grep "NOAA 15" "${NOAA_HOME}"/predict/weather.txt -A 2 > "${NOAA_HOME}"/predict/weather.tle
 grep "NOAA 18" "${NOAA_HOME}"/predict/weather.txt -A 2 >> "${NOAA_HOME}"/predict/weather.tle
 grep "NOAA 19" "${NOAA_HOME}"/predict/weather.txt -A 2 >> "${NOAA_HOME}"/predict/weather.tle
 grep "METEOR-M 2" "${NOAA_HOME}"/predict/weather.txt -A 2 >> "${NOAA_HOME}"/predict/weather.tle
-# grep "ZARYA" "${NOAA_HOME}"/predict/amateur.txt -A 2 > "${NOAA_HOME}"/predict/amateur.tle
+if [ "$SCHEDULE_ISS" == "true" ]; then
+    grep "ZARYA" "${NOAA_HOME}"/predict/amateur.txt -A 2 > "${NOAA_HOME}"/predict/amateur.tle
+fi
 
 #Remove all AT jobs
 for i in $(atq | awk '{print $1}');do atrm "$i";done
 
 #Schedule Satellite Passes:
-#"${NOAA_HOME}"/schedule_iss.sh "ISS (ZARYA)" 145.8000
+if [ "$SCHEDULE_ISS" == "true" ]; then
+    "${NOAA_HOME}"/schedule_iss.sh "ISS (ZARYA)" 145.8000
+fi
 "${NOAA_HOME}"/schedule_meteor.sh "METEOR-M 2" 137.1000
 "${NOAA_HOME}"/schedule_sat.sh "NOAA 19" 137.1000
 "${NOAA_HOME}"/schedule_sat.sh "NOAA 18" 137.9125

schedule_iss.sh

@@ -10,8 +10,6 @@ fi
 . "$HOME/.noaa.conf"
 . "$NOAA_HOME/common.sh"
 
-SAT_MIN_ELEV=10
-
 PREDICTION_START=$(/usr/bin/predict -t "${NOAA_HOME}"/predict/amateur.tle -p "${1}" | head -1)
 PREDICTION_END=$(/usr/bin/predict -t "${NOAA_HOME}"/predict/amateur.tle -p "${1}" | tail -1)
 
@@ -33,6 +31,7 @@ while [ "$(date --date="@${var2}" +%D)" = "$(date +%D)" ]; do
 		echo ${SATNAME} "${OUTDATE}" "$MAXELEV"
 		echo "${NOAA_HOME}/receive_iss.sh \"${1}\" $2 ISS${OUTDATE} "${NOAA_HOME}"/predict/amateur.tle \
 ${var1} ${TIMER} ${MAXELEV}" | at "$(date --date="TZ=\"UTC\" ${START_TIME}" +"%H:%M %D")"
+		sqlite3 /home/pi/raspberry-noaa/panel.db "insert or replace into predict_passes (sat_name,pass_start,pass_end,max_elev,is_active) values (\"$SATNAME\",$var1,$var2,$MAXELEV,1);"
 	fi
 	NEXTPREDICT=$(expr "${var2}" + 60)
 	PREDICTION_START=$(/usr/bin/predict -t "${NOAA_HOME}"/predict/amateur.tle -p "${1}" "${NEXTPREDICT}" | head -1)

templates/noaa.conf

@@ -12,3 +12,4 @@ BIAS_TEE="enable_bias_tee"
 DELETE_AUDIO="true"
 FLIP_METEOR_IMG="true"
 GAIN=50
+SCHEDULE_ISS="false"

templates/webpanel/Model/Conn.php

@@ -28,7 +28,7 @@
 
     public function getImages($page, $img_per_page) {
       $query = $this->con->prepare("SELECT  decoded_passes.id, predict_passes.pass_start, 
-                                            file_path, is_noaa, predict_passes.sat_name, predict_passes.max_elev 
+                                            file_path, sat_type, predict_passes.sat_name, predict_passes.max_elev 
                                             FROM decoded_passes INNER JOIN predict_passes 
                                             ON predict_passes.pass_start = decoded_passes.pass_start
                                             ORDER BY decoded_passes.pass_start DESC LIMIT ? OFFSET ?;");
@@ -45,19 +45,27 @@
     }
 
     public function getEnhacements($id) {
-      $query = $this->con->prepare('SELECT  daylight_pass, is_noaa
+      $query = $this->con->prepare('SELECT  daylight_pass, sat_type, img_count
                                             FROM decoded_passes WHERE id = ?;');
       $query->bindValue(1, $id);
       $result = $query->execute();
       $pass = $result->fetchArray();
-      if ($pass['is_noaa'] == 0) {
-      $enhacements = ['-122-rectified.jpg'];
-      } else {
-        if ($pass['daylight_pass'] == 1) {
-          $enhacements = ['-ZA.jpg','-MCIR.jpg','-MCIR-precip.jpg','-MSA.jpg','-MSA-precip.jpg','-HVC.jpg','-HVC-precip.jpg','-HVCT.jpg','-HVCT-precip.jpg'];
-        } else {
-          $enhacements = ['-ZA.jpg','-MCIR.jpg','-MCIR-precip.jpg'];
-        }
+      switch($pass['sat_type']) {
+        case 0: // Meteor-M2
+          $enhacements = ['-122-rectified.jpg'];
+          break;
+        case 1: // NOAA
+          if ($pass['daylight_pass'] == 1) {
+            $enhacements = ['-ZA.jpg','-MCIR.jpg','-MCIR-precip.jpg','-MSA.jpg','-MSA-precip.jpg','-HVC.jpg','-HVC-precip.jpg','-HVCT.jpg','-HVCT-precip.jpg'];
+          } else {
+            $enhacements = ['-ZA.jpg','-MCIR.jpg','-MCIR-precip.jpg'];
+          }
+          break;
+        case 2: // ISS
+          for ($x = 0; $x <= $pass['img_count']-1; $x++) {
+            $enhacements[] = "-$x.png";
+          }
+          break;
       }
       return $enhacements;
     }

templates/webpanel/Views/V_viewLastImages.php

@@ -20,10 +20,16 @@
           echo "<tr>";
           $col_count=1;
         }
-        if ($image['is_noaa'] == true) {
-          $ending = "-MCIR.jpg";
-        } else {
-          $ending = "-122-rectified.jpg";
+        switch($image['sat_type']) {
+          case 0: // Meteor-M2
+            $ending = "-122-rectified.jpg";
+            break;
+          case 1: // NOAA
+            $ending = "-MCIR.jpg";
+            break;
+          case 2: // ISS
+            $ending = "-0.png";
+            break;
         }
         echo "<td><div id =\"satimgdiv\"><a href=". "detail.php?id=" . $image['id'] ."><img id=\"satimg\" src=". $baseurl . "thumb/" . $image['file_path'] . $ending ."></img></a></div>";
         echo "<ul><li>". $image['sat_name'] ."</li>";

templates/webpanel_schema.sql

@@ -10,5 +10,5 @@ CREATE TABLE decoded_passes(
     id integer primary key autoincrement,
     pass_start integer,
     file_path text not null,
-    daylight_pass boolean, is_noaa boolean,
+    daylight_pass boolean, is_noaa boolean, sat_type integer, img_count integer,
     foreign key(pass_start) references passes(pass_start));