From 74ff51b035d815f5a19e1a0ac220c350f383ba25 Mon Sep 17 00:00:00 2001 From: MassiveBox Date: Wed, 4 Jan 2023 15:10:45 +0100 Subject: [PATCH] SQLite Improvements Turns out HomeAssistant only returns 10 days'data byault. This is a problem that we will havsoon. Now the cache doesn't reset eh time it only if some data is missing. --- .gitignore | 0 .woodpecker.yml | 0 Dockerfile | 0 LICENSE | 0 README.md | 0 api.go | 102 +++++++++++++++--- cache.go | 68 ++++++++---- config.go | 0 go.mod | 0 http.go | 10 +- main.go | 0 templates/default/accuracy-notice.html | 0 templates/default/admin.html | 0 templates/default/assets/bitcoin.svg | 0 templates/default/assets/chartjs/chart.js | 0 templates/default/assets/custom.css | 0 templates/default/assets/favicon.ico | Bin templates/default/assets/light-bulb.svg | 0 templates/default/assets/logo.svg | 0 templates/default/assets/oven.svg | 0 .../default/assets/picnic/picnic.min.css | 0 templates/default/assets/washing.svg | 0 templates/default/base.html | 0 templates/default/config-error.html | 0 templates/default/index.html | 0 templates/default/login.html | 0 templates/default/restart.html | 0 27 files changed, 140 insertions(+), 40 deletions(-) mode change 100644 => 100755 .gitignore mode change 100644 => 100755 .woodpecker.yml mode change 100644 => 100755 Dockerfile mode change 100644 => 100755 LICENSE mode change 100644 => 100755 README.md mode change 100644 => 100755 api.go mode change 100644 => 100755 cache.go mode change 100644 => 100755 config.go mode change 100644 => 100755 go.mod mode change 100644 => 100755 http.go mode change 100644 => 100755 main.go mode change 100644 => 100755 templates/default/accuracy-notice.html mode change 100644 => 100755 templates/default/admin.html mode change 100644 => 100755 templates/default/assets/bitcoin.svg mode change 100644 => 100755 templates/default/assets/chartjs/chart.js mode change 100644 => 100755 templates/default/assets/custom.css mode change 100644 => 100755 templates/default/assets/favicon.ico mode change 100644 => 100755 templates/default/assets/light-bulb.svg mode change 100644 => 100755 templates/default/assets/logo.svg mode change 100644 => 100755 templates/default/assets/oven.svg mode change 100644 => 100755 templates/default/assets/picnic/picnic.min.css mode change 100644 => 100755 templates/default/assets/washing.svg mode change 100644 => 100755 templates/default/base.html mode change 100644 => 100755 templates/default/config-error.html mode change 100644 => 100755 templates/default/index.html mode change 100644 => 100755 templates/default/login.html mode change 100644 => 100755 templates/default/restart.html diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/.woodpecker.yml b/.woodpecker.yml old mode 100644 new mode 100755 diff --git a/Dockerfile b/Dockerfile old mode 100644 new mode 100755 diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/api.go b/api.go old mode 100644 new mode 100755 index 877a4e8..73cc4d1 --- a/api.go +++ b/api.go @@ -12,11 +12,16 @@ import ( "time" ) -type HistoryResult [][]struct { +type HistoryResult []struct { State string `json:"state"` LastUpdated time.Time `json:"last_updated"` } +func dayStart(t time.Time) time.Time { + hours, minutes, seconds := t.Clock() + return t.Add(-(time.Duration(hours)*time.Hour + time.Duration(minutes)*time.Minute + time.Duration(seconds)*time.Second)) +} + func (config Config) queryHistory(entityID string, startTime, endTime time.Time) (HistoryResult, error) { req, err := http.NewRequest("GET", config.HomeAssistant.BaseURL+ @@ -46,13 +51,27 @@ func (config Config) queryHistory(entityID string, startTime, endTime time.Time) return HistoryResult{}, err } - var result HistoryResult + var result []HistoryResult err = json.Unmarshal(body, &result) if err != nil { - return result, err + return HistoryResult{}, err } - return result, nil + if len(result) != 1 { + return HistoryResult{}, nil + } + + return result[0], nil + +} + +// t can be any time during the desired day. +func (config Config) getDayHistory(entityID string, t time.Time) (HistoryResult, error) { + + hours, minutes, seconds := t.Clock() + endTime := t.Add(time.Duration(23-hours)*time.Hour + time.Duration(59-minutes)*time.Minute + time.Duration(59-seconds)*time.Second) + + return config.queryHistory(entityID, dayStart(t), endTime) } @@ -65,7 +84,33 @@ type DayData struct { Low float32 } -func (config Config) historyAverageAndConvertToGreen(entityID string, startTime, endTime time.Time) ([]DayData, error) { +func (config Config) historyAverageAndConvertToGreen(entityID string, t time.Time) (DayData, error) { + + history, err := config.getDayHistory(entityID, t) + if err != nil { + return DayData{}, err + } + + var day DayData + + for _, historyChange := range history { + + val, err := strconv.ParseFloat(historyChange.State, 32) + if err != nil { + continue + } + + day.Value += float32(val) + day.Measurements++ + + } + + day.Value = 100 - (day.Value / float32(day.Measurements)) + return day, nil + +} + +func (config Config) historyBulkAverageAndConvertToGreen(entityID string, startTime, endTime time.Time) ([]DayData, error) { history, err := config.queryHistory(entityID, startTime, endTime) if err != nil { @@ -74,7 +119,7 @@ func (config Config) historyAverageAndConvertToGreen(entityID string, startTime, var days []DayData - for _, historyChange := range history[0] { + for _, historyChange := range history { val, err := strconv.ParseFloat(historyChange.State, 32) if err != nil { continue @@ -93,7 +138,7 @@ func (config Config) historyAverageAndConvertToGreen(entityID string, startTime, if !found { days = append(days, DayData{ DayNumber: dayNo, - DayTime: historyChange.LastUpdated.Local(), + DayTime: dayStart(historyChange.LastUpdated.Local()), Measurements: 1, Value: value, }) @@ -112,7 +157,38 @@ func (config Config) historyAverageAndConvertToGreen(entityID string, startTime, } -func (config Config) historyDelta(entityID string, startTime, endTime time.Time) ([]DayData, error) { +func (config Config) historyDelta(entityID string, t time.Time) (DayData, error) { + + history, err := config.getDayHistory(entityID, t) + if err != nil { + return DayData{}, err + } + + var day DayData + + for _, historyChange := range history { + + val, err := strconv.ParseFloat(historyChange.State, 32) + if err != nil { + continue + } + value := float32(val) + + if value > day.High { + day.High = value + } + if value < day.Low || day.Low == 0 { + day.Low = value + } + + } + + day.Value = day.High - day.Low + return day, nil + +} + +func (config Config) historyBulkDelta(entityID string, startTime, endTime time.Time) ([]DayData, error) { history, err := config.queryHistory(entityID, startTime, endTime) if err != nil { @@ -121,7 +197,7 @@ func (config Config) historyDelta(entityID string, startTime, endTime time.Time) var days []DayData - for _, historyChange := range history[0] { + for _, historyChange := range history { if historyChange.State != "off" { val, err := strconv.ParseFloat(historyChange.State, 32) if err != nil { @@ -145,7 +221,7 @@ func (config Config) historyDelta(entityID string, startTime, endTime time.Time) if !found { days = append(days, DayData{ DayNumber: dayNo, - DayTime: historyChange.LastUpdated.Local(), + DayTime: dayStart(historyChange.LastUpdated.Local()), Value: value, }) } @@ -185,7 +261,7 @@ func fillMissing(days []DayData, startTime, endTime time.Time) []DayData { fakeTime := previousDay.Add(time.Duration(24*i) * time.Hour) ret = append(ret, DayData{ DayNumber: fakeTime.Day(), - DayTime: fakeTime, + DayTime: dayStart(fakeTime), Value: previousValue, }) } @@ -210,7 +286,7 @@ func fillMissing(days []DayData, startTime, endTime time.Time) []DayData { fakeTime := previousDay.Add(time.Duration(24*i) * time.Hour) ret = append(ret, DayData{ DayNumber: fakeTime.Day(), - DayTime: fakeTime, + DayTime: dayStart(fakeTime), Value: previousValue, }) } @@ -224,7 +300,7 @@ func fillMissing(days []DayData, startTime, endTime time.Time) []DayData { ret = append([]DayData{ { DayNumber: fakeTime.Day(), - DayTime: fakeTime, + DayTime: dayStart(fakeTime), Value: 0, }, }, ret...) diff --git a/cache.go b/cache.go old mode 100644 new mode 100755 index 2f528b3..e0f773d --- a/cache.go +++ b/cache.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "time" ) @@ -14,45 +15,72 @@ type CacheData []CacheEntry func (config Config) updateCache() { - // in order to avoid querying and storing each day's data from 0001-01-01 in future versions - if config.HomeAssistant.InstallationDate.IsZero() { - return - } - - now := time.Now() - h, m, s := now.Clock() - start := now.AddDate(0, 0, -7).Add(-(time.Duration(h)*time.Hour + time.Duration(m)*time.Minute + time.Duration(s)*time.Second)) - - greenEnergyPercentage, err := config.historyAverageAndConvertToGreen(config.Sensors.FossilPercentage, start, now) + greenEnergyPercentage, err := config.historyAverageAndConvertToGreen(config.Sensors.FossilPercentage, time.Now()) if err != nil { - fmt.Println("Error updating cached data for FossilPercentage -" + err.Error()) return } - historyPolledSmartEnergySummation, err := config.historyDelta(config.Sensors.PolledSmartEnergySummation, start, now) + historyPolledSmartEnergySummation, err := config.historyDelta(config.Sensors.PolledSmartEnergySummation, time.Now()) if err != nil { - fmt.Println("Error updating cached data for PolledSmartEnergySummation -" + err.Error()) return } - _, err = config.db.Exec("DELETE FROM cache") + config.db.Exec("INSERT INTO cache(time,green_energy_percentage,energy_consumption) VALUES (?,?,?);", dayStart(time.Now()).Unix(), greenEnergyPercentage.Value, historyPolledSmartEnergySummation.Value) + + cached, err := config.readCache() if err != nil { - fmt.Println("Error deleting previous records to cache: -", err.Error()) return } - - for key, day := range greenEnergyPercentage { - _, err := config.db.Exec("INSERT INTO cache(time,green_energy_percentage,energy_consumption) VALUES (?,?,?);", day.DayTime.Unix(), greenEnergyPercentage[key].Value, historyPolledSmartEnergySummation[key].Value) + if len(cached) != 8 && time.Now().Sub(config.HomeAssistant.InstallationDate) > 8*time.Hour*24 { + err := config.refreshCacheFromPast(time.Now().Add(-8 * time.Hour * 24)) if err != nil { - fmt.Println("Error adding record to cache: -", err.Error()) + fmt.Println("Error refreshing cache", err.Error()) return } } } +func (config Config) refreshCacheFromInstall() error { + return config.refreshCacheFromPast(config.HomeAssistant.InstallationDate) +} + +func (config Config) refreshCacheFromPast(pastTime time.Time) error { + + // in order to avoid querying and storing each day's data from 0001-01-01 in future versions + if config.HomeAssistant.InstallationDate.IsZero() { + return errors.New("installation date not set") + } + + greenEnergyPercentage, err := config.historyBulkAverageAndConvertToGreen(config.Sensors.FossilPercentage, pastTime, time.Now()) + if err != nil { + return err + } + historyPolledSmartEnergySummation, err := config.historyBulkDelta(config.Sensors.PolledSmartEnergySummation, pastTime, time.Now()) + if err != nil { + return err + } + + _, err = config.db.Exec("DELETE FROM cache") + if err != nil { + return err + } + + for key, day := range greenEnergyPercentage { + _, err := config.db.Exec("INSERT INTO cache(time,green_energy_percentage,energy_consumption) VALUES (?,?,?);", day.DayTime.Unix(), greenEnergyPercentage[key].Value, historyPolledSmartEnergySummation[key].Value) + if err != nil { + return err + } + } + + return nil + +} + func (config Config) readCache() (CacheData, error) { - rows, err := config.db.Query("SELECT time, green_energy_percentage, energy_consumption FROM cache") + start := dayStart(time.Now()).AddDate(0, 0, -8) + + rows, err := config.db.Query("SELECT time, green_energy_percentage, energy_consumption FROM cache WHERE time > ?", start.Unix()) if err != nil { return CacheData{}, err } diff --git a/config.go b/config.go old mode 100644 new mode 100755 diff --git a/go.mod b/go.mod old mode 100644 new mode 100755 diff --git a/http.go b/http.go old mode 100644 new mode 100755 index 5a54713..64376ef --- a/http.go +++ b/http.go @@ -118,7 +118,7 @@ func (config Config) saveAdminForm(c *fiber.Ctx) error { } form := Config{ - HomeAssistant: HomeAssistant{ /*BaseURL to be filled later*/ ApiKey: c.FormValue("api_key"), InstallationDate: parsedTime}, + HomeAssistant: HomeAssistant{ /*BaseURL to be filled later*/ ApiKey: c.FormValue("api_key"), InstallationDate: dayStart(parsedTime)}, Sensors: Sensors{PolledSmartEnergySummation: c.FormValue("polled_smart_energy_summation"), FossilPercentage: c.FormValue("fossil_percentage")}, Administrator: Administrator{Username: c.FormValue("username") /*PasswordHash to be filled later*/}, Dashboard: Dashboard{Theme: c.FormValue("theme"), Name: c.FormValue("name"), HeaderLinks: config.Dashboard.HeaderLinks, FooterLinks: config.Dashboard.FooterLinks}, @@ -140,12 +140,8 @@ func (config Config) saveAdminForm(c *fiber.Ctx) error { return errors.New("No changes from previous config.") } - // in order to test if ha base URL, API key and entity IDs are correct we try fetching the devices history - _, err = form.queryHistory(form.Sensors.FossilPercentage, time.Now().Add(-5*time.Minute), time.Now()) - if err != nil { - return err - } - _, err = form.queryHistory(form.Sensors.PolledSmartEnergySummation, time.Now().Add(-5*time.Minute), time.Now()) + form.db = config.db + err = form.refreshCacheFromInstall() if err != nil { return err } diff --git a/main.go b/main.go old mode 100644 new mode 100755 diff --git a/templates/default/accuracy-notice.html b/templates/default/accuracy-notice.html old mode 100644 new mode 100755 diff --git a/templates/default/admin.html b/templates/default/admin.html old mode 100644 new mode 100755 diff --git a/templates/default/assets/bitcoin.svg b/templates/default/assets/bitcoin.svg old mode 100644 new mode 100755 diff --git a/templates/default/assets/chartjs/chart.js b/templates/default/assets/chartjs/chart.js old mode 100644 new mode 100755 diff --git a/templates/default/assets/custom.css b/templates/default/assets/custom.css old mode 100644 new mode 100755 diff --git a/templates/default/assets/favicon.ico b/templates/default/assets/favicon.ico old mode 100644 new mode 100755 diff --git a/templates/default/assets/light-bulb.svg b/templates/default/assets/light-bulb.svg old mode 100644 new mode 100755 diff --git a/templates/default/assets/logo.svg b/templates/default/assets/logo.svg old mode 100644 new mode 100755 diff --git a/templates/default/assets/oven.svg b/templates/default/assets/oven.svg old mode 100644 new mode 100755 diff --git a/templates/default/assets/picnic/picnic.min.css b/templates/default/assets/picnic/picnic.min.css old mode 100644 new mode 100755 diff --git a/templates/default/assets/washing.svg b/templates/default/assets/washing.svg old mode 100644 new mode 100755 diff --git a/templates/default/base.html b/templates/default/base.html old mode 100644 new mode 100755 diff --git a/templates/default/config-error.html b/templates/default/config-error.html old mode 100644 new mode 100755 diff --git a/templates/default/index.html b/templates/default/index.html old mode 100644 new mode 100755 diff --git a/templates/default/login.html b/templates/default/login.html old mode 100644 new mode 100755 diff --git a/templates/default/restart.html b/templates/default/restart.html old mode 100644 new mode 100755