From 52ba0ea4c1eecdc07d52648b523ec4d45c40b84a Mon Sep 17 00:00:00 2001 From: MassiveBox Date: Wed, 4 Jan 2023 15:57:16 +0100 Subject: [PATCH] Improve SQLite - Now the cache isn't cleared and fetched from zero each update, but only when there's missing information for a day - Fixed a bug that prevented new users from saving changes in the config NOTE: Turns out that HomeAssistant's API only returns data for the last 10 days. Looks like we will have to improve the caching system to work around this. --- api.go | 102 +++++++++++++++++++++++++++++++++++++++++++++++------- cache.go | 68 +++++++++++++++++++++++++----------- config.go | 31 +++++++++-------- http.go | 10 ++---- 4 files changed, 156 insertions(+), 55 deletions(-) diff --git a/api.go b/api.go index 877a4e8..73cc4d1 100644 --- a/api.go +++ b/api.go @@ -12,11 +12,16 @@ import ( "time" ) -type HistoryResult [][]struct { +type HistoryResult []struct { State string `json:"state"` LastUpdated time.Time `json:"last_updated"` } +func dayStart(t time.Time) time.Time { + hours, minutes, seconds := t.Clock() + return t.Add(-(time.Duration(hours)*time.Hour + time.Duration(minutes)*time.Minute + time.Duration(seconds)*time.Second)) +} + func (config Config) queryHistory(entityID string, startTime, endTime time.Time) (HistoryResult, error) { req, err := http.NewRequest("GET", config.HomeAssistant.BaseURL+ @@ -46,13 +51,27 @@ func (config Config) queryHistory(entityID string, startTime, endTime time.Time) return HistoryResult{}, err } - var result HistoryResult + var result []HistoryResult err = json.Unmarshal(body, &result) if err != nil { - return result, err + return HistoryResult{}, err } - return result, nil + if len(result) != 1 { + return HistoryResult{}, nil + } + + return result[0], nil + +} + +// t can be any time during the desired day. +func (config Config) getDayHistory(entityID string, t time.Time) (HistoryResult, error) { + + hours, minutes, seconds := t.Clock() + endTime := t.Add(time.Duration(23-hours)*time.Hour + time.Duration(59-minutes)*time.Minute + time.Duration(59-seconds)*time.Second) + + return config.queryHistory(entityID, dayStart(t), endTime) } @@ -65,7 +84,33 @@ type DayData struct { Low float32 } -func (config Config) historyAverageAndConvertToGreen(entityID string, startTime, endTime time.Time) ([]DayData, error) { +func (config Config) historyAverageAndConvertToGreen(entityID string, t time.Time) (DayData, error) { + + history, err := config.getDayHistory(entityID, t) + if err != nil { + return DayData{}, err + } + + var day DayData + + for _, historyChange := range history { + + val, err := strconv.ParseFloat(historyChange.State, 32) + if err != nil { + continue + } + + day.Value += float32(val) + day.Measurements++ + + } + + day.Value = 100 - (day.Value / float32(day.Measurements)) + return day, nil + +} + +func (config Config) historyBulkAverageAndConvertToGreen(entityID string, startTime, endTime time.Time) ([]DayData, error) { history, err := config.queryHistory(entityID, startTime, endTime) if err != nil { @@ -74,7 +119,7 @@ func (config Config) historyAverageAndConvertToGreen(entityID string, startTime, var days []DayData - for _, historyChange := range history[0] { + for _, historyChange := range history { val, err := strconv.ParseFloat(historyChange.State, 32) if err != nil { continue @@ -93,7 +138,7 @@ func (config Config) historyAverageAndConvertToGreen(entityID string, startTime, if !found { days = append(days, DayData{ DayNumber: dayNo, - DayTime: historyChange.LastUpdated.Local(), + DayTime: dayStart(historyChange.LastUpdated.Local()), Measurements: 1, Value: value, }) @@ -112,7 +157,38 @@ func (config Config) historyAverageAndConvertToGreen(entityID string, startTime, } -func (config Config) historyDelta(entityID string, startTime, endTime time.Time) ([]DayData, error) { +func (config Config) historyDelta(entityID string, t time.Time) (DayData, error) { + + history, err := config.getDayHistory(entityID, t) + if err != nil { + return DayData{}, err + } + + var day DayData + + for _, historyChange := range history { + + val, err := strconv.ParseFloat(historyChange.State, 32) + if err != nil { + continue + } + value := float32(val) + + if value > day.High { + day.High = value + } + if value < day.Low || day.Low == 0 { + day.Low = value + } + + } + + day.Value = day.High - day.Low + return day, nil + +} + +func (config Config) historyBulkDelta(entityID string, startTime, endTime time.Time) ([]DayData, error) { history, err := config.queryHistory(entityID, startTime, endTime) if err != nil { @@ -121,7 +197,7 @@ func (config Config) historyDelta(entityID string, startTime, endTime time.Time) var days []DayData - for _, historyChange := range history[0] { + for _, historyChange := range history { if historyChange.State != "off" { val, err := strconv.ParseFloat(historyChange.State, 32) if err != nil { @@ -145,7 +221,7 @@ func (config Config) historyDelta(entityID string, startTime, endTime time.Time) if !found { days = append(days, DayData{ DayNumber: dayNo, - DayTime: historyChange.LastUpdated.Local(), + DayTime: dayStart(historyChange.LastUpdated.Local()), Value: value, }) } @@ -185,7 +261,7 @@ func fillMissing(days []DayData, startTime, endTime time.Time) []DayData { fakeTime := previousDay.Add(time.Duration(24*i) * time.Hour) ret = append(ret, DayData{ DayNumber: fakeTime.Day(), - DayTime: fakeTime, + DayTime: dayStart(fakeTime), Value: previousValue, }) } @@ -210,7 +286,7 @@ func fillMissing(days []DayData, startTime, endTime time.Time) []DayData { fakeTime := previousDay.Add(time.Duration(24*i) * time.Hour) ret = append(ret, DayData{ DayNumber: fakeTime.Day(), - DayTime: fakeTime, + DayTime: dayStart(fakeTime), Value: previousValue, }) } @@ -224,7 +300,7 @@ func fillMissing(days []DayData, startTime, endTime time.Time) []DayData { ret = append([]DayData{ { DayNumber: fakeTime.Day(), - DayTime: fakeTime, + DayTime: dayStart(fakeTime), Value: 0, }, }, ret...) diff --git a/cache.go b/cache.go index 2f528b3..e0f773d 100644 --- a/cache.go +++ b/cache.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "time" ) @@ -14,45 +15,72 @@ type CacheData []CacheEntry func (config Config) updateCache() { - // in order to avoid querying and storing each day's data from 0001-01-01 in future versions - if config.HomeAssistant.InstallationDate.IsZero() { - return - } - - now := time.Now() - h, m, s := now.Clock() - start := now.AddDate(0, 0, -7).Add(-(time.Duration(h)*time.Hour + time.Duration(m)*time.Minute + time.Duration(s)*time.Second)) - - greenEnergyPercentage, err := config.historyAverageAndConvertToGreen(config.Sensors.FossilPercentage, start, now) + greenEnergyPercentage, err := config.historyAverageAndConvertToGreen(config.Sensors.FossilPercentage, time.Now()) if err != nil { - fmt.Println("Error updating cached data for FossilPercentage -" + err.Error()) return } - historyPolledSmartEnergySummation, err := config.historyDelta(config.Sensors.PolledSmartEnergySummation, start, now) + historyPolledSmartEnergySummation, err := config.historyDelta(config.Sensors.PolledSmartEnergySummation, time.Now()) if err != nil { - fmt.Println("Error updating cached data for PolledSmartEnergySummation -" + err.Error()) return } - _, err = config.db.Exec("DELETE FROM cache") + config.db.Exec("INSERT INTO cache(time,green_energy_percentage,energy_consumption) VALUES (?,?,?);", dayStart(time.Now()).Unix(), greenEnergyPercentage.Value, historyPolledSmartEnergySummation.Value) + + cached, err := config.readCache() if err != nil { - fmt.Println("Error deleting previous records to cache: -", err.Error()) return } - - for key, day := range greenEnergyPercentage { - _, err := config.db.Exec("INSERT INTO cache(time,green_energy_percentage,energy_consumption) VALUES (?,?,?);", day.DayTime.Unix(), greenEnergyPercentage[key].Value, historyPolledSmartEnergySummation[key].Value) + if len(cached) != 8 && time.Now().Sub(config.HomeAssistant.InstallationDate) > 8*time.Hour*24 { + err := config.refreshCacheFromPast(time.Now().Add(-8 * time.Hour * 24)) if err != nil { - fmt.Println("Error adding record to cache: -", err.Error()) + fmt.Println("Error refreshing cache", err.Error()) return } } } +func (config Config) refreshCacheFromInstall() error { + return config.refreshCacheFromPast(config.HomeAssistant.InstallationDate) +} + +func (config Config) refreshCacheFromPast(pastTime time.Time) error { + + // in order to avoid querying and storing each day's data from 0001-01-01 in future versions + if config.HomeAssistant.InstallationDate.IsZero() { + return errors.New("installation date not set") + } + + greenEnergyPercentage, err := config.historyBulkAverageAndConvertToGreen(config.Sensors.FossilPercentage, pastTime, time.Now()) + if err != nil { + return err + } + historyPolledSmartEnergySummation, err := config.historyBulkDelta(config.Sensors.PolledSmartEnergySummation, pastTime, time.Now()) + if err != nil { + return err + } + + _, err = config.db.Exec("DELETE FROM cache") + if err != nil { + return err + } + + for key, day := range greenEnergyPercentage { + _, err := config.db.Exec("INSERT INTO cache(time,green_energy_percentage,energy_consumption) VALUES (?,?,?);", day.DayTime.Unix(), greenEnergyPercentage[key].Value, historyPolledSmartEnergySummation[key].Value) + if err != nil { + return err + } + } + + return nil + +} + func (config Config) readCache() (CacheData, error) { - rows, err := config.db.Query("SELECT time, green_energy_percentage, energy_consumption FROM cache") + start := dayStart(time.Now()).AddDate(0, 0, -8) + + rows, err := config.db.Query("SELECT time, green_energy_percentage, energy_consumption FROM cache WHERE time > ?", start.Unix()) if err != nil { return CacheData{}, err } diff --git a/config.go b/config.go index 89b399e..2d01781 100644 --- a/config.go +++ b/config.go @@ -64,6 +64,21 @@ func formatURL(url string) (string, error) { func loadConfig() (config Config, err error, isFirstRun bool) { + db, err := sql.Open("sqlite3", "./database.db") + if err != nil { + return Config{}, err, false + } + + _, err = db.Exec(`CREATE TABLE IF NOT EXISTS "cache" ( + "time" NUMERIC NOT NULL, + "green_energy_percentage" REAL NOT NULL, + "energy_consumption" REAL NOT NULL, + PRIMARY KEY("time") + );`) + if err != nil { + return Config{}, err, false + } + var defaultConfig = Config{} defaultConfig.Dashboard.Theme = "default" defaultConfig.Dashboard.Name = "EcoDash" @@ -76,6 +91,7 @@ func loadConfig() (config Config, err error, isFirstRun bool) { NewTab: true, Primary: true, }) + defaultConfig.db = db data, err := os.ReadFile("config.json") if err != nil { @@ -96,23 +112,8 @@ func loadConfig() (config Config, err error, isFirstRun bool) { if err != nil { return Config{}, err, false } - - db, err := sql.Open("sqlite3", "./database.db") - if err != nil { - return Config{}, err, false - } conf.db = db - _, err = db.Exec(`CREATE TABLE IF NOT EXISTS "cache" ( - "time" NUMERIC NOT NULL, - "green_energy_percentage" REAL NOT NULL, - "energy_consumption" REAL NOT NULL, - PRIMARY KEY("time") - );`) - if err != nil { - return Config{}, err, false - } - return conf, nil, false } diff --git a/http.go b/http.go index 5a54713..64376ef 100644 --- a/http.go +++ b/http.go @@ -118,7 +118,7 @@ func (config Config) saveAdminForm(c *fiber.Ctx) error { } form := Config{ - HomeAssistant: HomeAssistant{ /*BaseURL to be filled later*/ ApiKey: c.FormValue("api_key"), InstallationDate: parsedTime}, + HomeAssistant: HomeAssistant{ /*BaseURL to be filled later*/ ApiKey: c.FormValue("api_key"), InstallationDate: dayStart(parsedTime)}, Sensors: Sensors{PolledSmartEnergySummation: c.FormValue("polled_smart_energy_summation"), FossilPercentage: c.FormValue("fossil_percentage")}, Administrator: Administrator{Username: c.FormValue("username") /*PasswordHash to be filled later*/}, Dashboard: Dashboard{Theme: c.FormValue("theme"), Name: c.FormValue("name"), HeaderLinks: config.Dashboard.HeaderLinks, FooterLinks: config.Dashboard.FooterLinks}, @@ -140,12 +140,8 @@ func (config Config) saveAdminForm(c *fiber.Ctx) error { return errors.New("No changes from previous config.") } - // in order to test if ha base URL, API key and entity IDs are correct we try fetching the devices history - _, err = form.queryHistory(form.Sensors.FossilPercentage, time.Now().Add(-5*time.Minute), time.Now()) - if err != nil { - return err - } - _, err = form.queryHistory(form.Sensors.PolledSmartEnergySummation, time.Now().Add(-5*time.Minute), time.Now()) + form.db = config.db + err = form.refreshCacheFromInstall() if err != nil { return err }