SQLite Improvements

Turns out HomeAssistant only returns 10 days'data byault. This is a problem that we will havsoon.
Now the cache doesn't reset eh time it only if some data is missing.
This commit is contained in:
MassiveBox 2023-01-04 15:10:45 +01:00
parent e9125b783c
commit 74ff51b035
27 changed files with 140 additions and 40 deletions

0
.gitignore vendored Normal file → Executable file
View File

0
.woodpecker.yml Normal file → Executable file
View File

0
Dockerfile Normal file → Executable file
View File

0
LICENSE Normal file → Executable file
View File

0
README.md Normal file → Executable file
View File

102
api.go Normal file → Executable file
View File

@ -12,11 +12,16 @@ import (
"time"
)
type HistoryResult [][]struct {
type HistoryResult []struct {
State string `json:"state"`
LastUpdated time.Time `json:"last_updated"`
}
func dayStart(t time.Time) time.Time {
hours, minutes, seconds := t.Clock()
return t.Add(-(time.Duration(hours)*time.Hour + time.Duration(minutes)*time.Minute + time.Duration(seconds)*time.Second))
}
func (config Config) queryHistory(entityID string, startTime, endTime time.Time) (HistoryResult, error) {
req, err := http.NewRequest("GET", config.HomeAssistant.BaseURL+
@ -46,13 +51,27 @@ func (config Config) queryHistory(entityID string, startTime, endTime time.Time)
return HistoryResult{}, err
}
var result HistoryResult
var result []HistoryResult
err = json.Unmarshal(body, &result)
if err != nil {
return result, err
return HistoryResult{}, err
}
return result, nil
if len(result) != 1 {
return HistoryResult{}, nil
}
return result[0], nil
}
// t can be any time during the desired day.
func (config Config) getDayHistory(entityID string, t time.Time) (HistoryResult, error) {
hours, minutes, seconds := t.Clock()
endTime := t.Add(time.Duration(23-hours)*time.Hour + time.Duration(59-minutes)*time.Minute + time.Duration(59-seconds)*time.Second)
return config.queryHistory(entityID, dayStart(t), endTime)
}
@ -65,7 +84,33 @@ type DayData struct {
Low float32
}
func (config Config) historyAverageAndConvertToGreen(entityID string, startTime, endTime time.Time) ([]DayData, error) {
func (config Config) historyAverageAndConvertToGreen(entityID string, t time.Time) (DayData, error) {
history, err := config.getDayHistory(entityID, t)
if err != nil {
return DayData{}, err
}
var day DayData
for _, historyChange := range history {
val, err := strconv.ParseFloat(historyChange.State, 32)
if err != nil {
continue
}
day.Value += float32(val)
day.Measurements++
}
day.Value = 100 - (day.Value / float32(day.Measurements))
return day, nil
}
func (config Config) historyBulkAverageAndConvertToGreen(entityID string, startTime, endTime time.Time) ([]DayData, error) {
history, err := config.queryHistory(entityID, startTime, endTime)
if err != nil {
@ -74,7 +119,7 @@ func (config Config) historyAverageAndConvertToGreen(entityID string, startTime,
var days []DayData
for _, historyChange := range history[0] {
for _, historyChange := range history {
val, err := strconv.ParseFloat(historyChange.State, 32)
if err != nil {
continue
@ -93,7 +138,7 @@ func (config Config) historyAverageAndConvertToGreen(entityID string, startTime,
if !found {
days = append(days, DayData{
DayNumber: dayNo,
DayTime: historyChange.LastUpdated.Local(),
DayTime: dayStart(historyChange.LastUpdated.Local()),
Measurements: 1,
Value: value,
})
@ -112,7 +157,38 @@ func (config Config) historyAverageAndConvertToGreen(entityID string, startTime,
}
func (config Config) historyDelta(entityID string, startTime, endTime time.Time) ([]DayData, error) {
func (config Config) historyDelta(entityID string, t time.Time) (DayData, error) {
history, err := config.getDayHistory(entityID, t)
if err != nil {
return DayData{}, err
}
var day DayData
for _, historyChange := range history {
val, err := strconv.ParseFloat(historyChange.State, 32)
if err != nil {
continue
}
value := float32(val)
if value > day.High {
day.High = value
}
if value < day.Low || day.Low == 0 {
day.Low = value
}
}
day.Value = day.High - day.Low
return day, nil
}
func (config Config) historyBulkDelta(entityID string, startTime, endTime time.Time) ([]DayData, error) {
history, err := config.queryHistory(entityID, startTime, endTime)
if err != nil {
@ -121,7 +197,7 @@ func (config Config) historyDelta(entityID string, startTime, endTime time.Time)
var days []DayData
for _, historyChange := range history[0] {
for _, historyChange := range history {
if historyChange.State != "off" {
val, err := strconv.ParseFloat(historyChange.State, 32)
if err != nil {
@ -145,7 +221,7 @@ func (config Config) historyDelta(entityID string, startTime, endTime time.Time)
if !found {
days = append(days, DayData{
DayNumber: dayNo,
DayTime: historyChange.LastUpdated.Local(),
DayTime: dayStart(historyChange.LastUpdated.Local()),
Value: value,
})
}
@ -185,7 +261,7 @@ func fillMissing(days []DayData, startTime, endTime time.Time) []DayData {
fakeTime := previousDay.Add(time.Duration(24*i) * time.Hour)
ret = append(ret, DayData{
DayNumber: fakeTime.Day(),
DayTime: fakeTime,
DayTime: dayStart(fakeTime),
Value: previousValue,
})
}
@ -210,7 +286,7 @@ func fillMissing(days []DayData, startTime, endTime time.Time) []DayData {
fakeTime := previousDay.Add(time.Duration(24*i) * time.Hour)
ret = append(ret, DayData{
DayNumber: fakeTime.Day(),
DayTime: fakeTime,
DayTime: dayStart(fakeTime),
Value: previousValue,
})
}
@ -224,7 +300,7 @@ func fillMissing(days []DayData, startTime, endTime time.Time) []DayData {
ret = append([]DayData{
{
DayNumber: fakeTime.Day(),
DayTime: fakeTime,
DayTime: dayStart(fakeTime),
Value: 0,
},
}, ret...)

68
cache.go Normal file → Executable file
View File

@ -1,6 +1,7 @@
package main
import (
"errors"
"fmt"
"time"
)
@ -14,45 +15,72 @@ type CacheData []CacheEntry
func (config Config) updateCache() {
// in order to avoid querying and storing each day's data from 0001-01-01 in future versions
if config.HomeAssistant.InstallationDate.IsZero() {
return
}
now := time.Now()
h, m, s := now.Clock()
start := now.AddDate(0, 0, -7).Add(-(time.Duration(h)*time.Hour + time.Duration(m)*time.Minute + time.Duration(s)*time.Second))
greenEnergyPercentage, err := config.historyAverageAndConvertToGreen(config.Sensors.FossilPercentage, start, now)
greenEnergyPercentage, err := config.historyAverageAndConvertToGreen(config.Sensors.FossilPercentage, time.Now())
if err != nil {
fmt.Println("Error updating cached data for FossilPercentage -" + err.Error())
return
}
historyPolledSmartEnergySummation, err := config.historyDelta(config.Sensors.PolledSmartEnergySummation, start, now)
historyPolledSmartEnergySummation, err := config.historyDelta(config.Sensors.PolledSmartEnergySummation, time.Now())
if err != nil {
fmt.Println("Error updating cached data for PolledSmartEnergySummation -" + err.Error())
return
}
_, err = config.db.Exec("DELETE FROM cache")
config.db.Exec("INSERT INTO cache(time,green_energy_percentage,energy_consumption) VALUES (?,?,?);", dayStart(time.Now()).Unix(), greenEnergyPercentage.Value, historyPolledSmartEnergySummation.Value)
cached, err := config.readCache()
if err != nil {
fmt.Println("Error deleting previous records to cache: -", err.Error())
return
}
for key, day := range greenEnergyPercentage {
_, err := config.db.Exec("INSERT INTO cache(time,green_energy_percentage,energy_consumption) VALUES (?,?,?);", day.DayTime.Unix(), greenEnergyPercentage[key].Value, historyPolledSmartEnergySummation[key].Value)
if len(cached) != 8 && time.Now().Sub(config.HomeAssistant.InstallationDate) > 8*time.Hour*24 {
err := config.refreshCacheFromPast(time.Now().Add(-8 * time.Hour * 24))
if err != nil {
fmt.Println("Error adding record to cache: -", err.Error())
fmt.Println("Error refreshing cache", err.Error())
return
}
}
}
func (config Config) refreshCacheFromInstall() error {
return config.refreshCacheFromPast(config.HomeAssistant.InstallationDate)
}
func (config Config) refreshCacheFromPast(pastTime time.Time) error {
// in order to avoid querying and storing each day's data from 0001-01-01 in future versions
if config.HomeAssistant.InstallationDate.IsZero() {
return errors.New("installation date not set")
}
greenEnergyPercentage, err := config.historyBulkAverageAndConvertToGreen(config.Sensors.FossilPercentage, pastTime, time.Now())
if err != nil {
return err
}
historyPolledSmartEnergySummation, err := config.historyBulkDelta(config.Sensors.PolledSmartEnergySummation, pastTime, time.Now())
if err != nil {
return err
}
_, err = config.db.Exec("DELETE FROM cache")
if err != nil {
return err
}
for key, day := range greenEnergyPercentage {
_, err := config.db.Exec("INSERT INTO cache(time,green_energy_percentage,energy_consumption) VALUES (?,?,?);", day.DayTime.Unix(), greenEnergyPercentage[key].Value, historyPolledSmartEnergySummation[key].Value)
if err != nil {
return err
}
}
return nil
}
func (config Config) readCache() (CacheData, error) {
rows, err := config.db.Query("SELECT time, green_energy_percentage, energy_consumption FROM cache")
start := dayStart(time.Now()).AddDate(0, 0, -8)
rows, err := config.db.Query("SELECT time, green_energy_percentage, energy_consumption FROM cache WHERE time > ?", start.Unix())
if err != nil {
return CacheData{}, err
}

0
config.go Normal file → Executable file
View File

0
go.mod Normal file → Executable file
View File

10
http.go Normal file → Executable file
View File

@ -118,7 +118,7 @@ func (config Config) saveAdminForm(c *fiber.Ctx) error {
}
form := Config{
HomeAssistant: HomeAssistant{ /*BaseURL to be filled later*/ ApiKey: c.FormValue("api_key"), InstallationDate: parsedTime},
HomeAssistant: HomeAssistant{ /*BaseURL to be filled later*/ ApiKey: c.FormValue("api_key"), InstallationDate: dayStart(parsedTime)},
Sensors: Sensors{PolledSmartEnergySummation: c.FormValue("polled_smart_energy_summation"), FossilPercentage: c.FormValue("fossil_percentage")},
Administrator: Administrator{Username: c.FormValue("username") /*PasswordHash to be filled later*/},
Dashboard: Dashboard{Theme: c.FormValue("theme"), Name: c.FormValue("name"), HeaderLinks: config.Dashboard.HeaderLinks, FooterLinks: config.Dashboard.FooterLinks},
@ -140,12 +140,8 @@ func (config Config) saveAdminForm(c *fiber.Ctx) error {
return errors.New("No changes from previous config.")
}
// in order to test if ha base URL, API key and entity IDs are correct we try fetching the devices history
_, err = form.queryHistory(form.Sensors.FossilPercentage, time.Now().Add(-5*time.Minute), time.Now())
if err != nil {
return err
}
_, err = form.queryHistory(form.Sensors.PolledSmartEnergySummation, time.Now().Add(-5*time.Minute), time.Now())
form.db = config.db
err = form.refreshCacheFromInstall()
if err != nil {
return err
}

0
main.go Normal file → Executable file
View File

0
templates/default/accuracy-notice.html Normal file → Executable file
View File

0
templates/default/admin.html Normal file → Executable file
View File

0
templates/default/assets/bitcoin.svg Normal file → Executable file
View File

Before

Width:  |  Height:  |  Size: 2.3 KiB

After

Width:  |  Height:  |  Size: 2.3 KiB

0
templates/default/assets/chartjs/chart.js Normal file → Executable file
View File

0
templates/default/assets/custom.css Normal file → Executable file
View File

0
templates/default/assets/favicon.ico Normal file → Executable file
View File

Before

Width:  |  Height:  |  Size: 4.6 KiB

After

Width:  |  Height:  |  Size: 4.6 KiB

0
templates/default/assets/light-bulb.svg Normal file → Executable file
View File

Before

Width:  |  Height:  |  Size: 2.6 KiB

After

Width:  |  Height:  |  Size: 2.6 KiB

0
templates/default/assets/logo.svg Normal file → Executable file
View File

Before

Width:  |  Height:  |  Size: 603 B

After

Width:  |  Height:  |  Size: 603 B

0
templates/default/assets/oven.svg Normal file → Executable file
View File

Before

Width:  |  Height:  |  Size: 3.5 KiB

After

Width:  |  Height:  |  Size: 3.5 KiB

0
templates/default/assets/picnic/picnic.min.css vendored Normal file → Executable file
View File

0
templates/default/assets/washing.svg Normal file → Executable file
View File

Before

Width:  |  Height:  |  Size: 4.7 KiB

After

Width:  |  Height:  |  Size: 4.7 KiB

0
templates/default/base.html Normal file → Executable file
View File

0
templates/default/config-error.html Normal file → Executable file
View File

0
templates/default/index.html Normal file → Executable file
View File

0
templates/default/login.html Normal file → Executable file
View File

0
templates/default/restart.html Normal file → Executable file
View File