Add safeguard on thermal sensor failures

pkg/temperature/ds18b20.go

@@ -34,12 +34,15 @@ var (
 	// ConfigUpdates receives the list of sensors to read
 	ConfigUpdates chan []string
 	sensors       []string
+	accErrTrigger int
+	accErrSensor  map[string]int
 )
 
 func Initialize() {
 	sensors = make([]string, 0)
 	C = make(chan TemperatureReading, 30)
 	ConfigUpdates = make(chan []string, 1)
+	accErrSensor = make(map[string]int)
 }
 
 func PollSensors(ctx context.Context, wg *sync.WaitGroup, readingInterval time.Duration) {
@@ -59,13 +62,26 @@ func PollSensors(ctx context.Context, wg *sync.WaitGroup, readingInterval time.D
 		case <-ticker.C:
 			start := time.Now()
 
+			if accErrTrigger > 60 {
+				log.Fatal("Thermal bulk read failed 60 times in a row -- terminating")
+			}
+
+			for _, sensor := range sensors {
+				if accErrSensor[sensor] > 60 {
+					log.Fatalf("Thermal sensor read of %v failed 60 times in a row -- terminating", sensor)
+				}
+			}
+
 			// Trigger a bulk read to start conversion on all sensors.
 			if err := triggerBulkRead(); err != nil {
+				accErrTrigger++
 				hub.CaptureException(err)
 				log.Print(err)
 				break
 			}
 
+			accErrTrigger = 0
+
 			// Ensure that we wait for sensors to convert data.
 			deltaSleep := sensorConversionTime - time.Since(start)
 			if deltaSleep > 0 {
@@ -96,11 +112,14 @@ func readSensors() {
 		dur := time.Since(start).Seconds()
 
 		if err != nil {
+			accErrSensor[sensor]++
 			hub.CaptureException(err)
 			log.Printf("Error reading temperature sensor %v: %v", sensor, err)
 			continue
 		}
 
+		accErrSensor[sensor] = 0
+
 		r := TemperatureReading{
 			Time:         time.Now(),
 			Sensor:       sensor,