Cold storage (#57)

* feat: Implement S3 integration for system save data migration and retrieval

* feat: Refactor system save data handling to use JSON and improve error logging

* feat: Update S3 migration schedule to hourly and limit old account retrieval to 3000 records

* fix: Clean up S3 migration code by removing unnecessary blank lines and improving logging messages
pull/50/merge
Frederico Santos 4 weeks ago committed by GitHub
parent 430496941c
commit ac360ccb1c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -18,15 +18,21 @@
package daily
import (
"bytes"
"context"
"crypto/md5"
"crypto/rand"
"encoding/base64"
"encoding/binary"
"encoding/json"
"fmt"
"log"
"os"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/pagefaultgames/rogueserver/db"
"github.com/robfig/cron/v3"
)
@ -34,8 +40,9 @@ import (
const secondsPerDay = 60 * 60 * 24
var (
scheduler = cron.New(cron.WithLocation(time.UTC))
secret []byte
scheduler = cron.New(cron.WithLocation(time.UTC))
s3scheduler = cron.New(cron.WithLocation(time.UTC))
secret []byte
)
func Init() error {
@ -84,6 +91,22 @@ func Init() error {
scheduler.Start()
if os.Getenv("AWS_ENDPOINT_URL_S3") == "" {
log.Printf("AWS_ENDPOINT_URL_S3 not set, skipping s3 migration")
return nil
}
_, err = s3scheduler.AddFunc("@hourly", func() {
time.Sleep(time.Second)
S3SaveMigration()
})
if err != nil {
return err
}
s3scheduler.Start()
return nil
}
@ -99,3 +122,39 @@ func deriveSeed(seedTime time.Time) []byte {
return hashedSeed[:]
}
func S3SaveMigration() {
cfg, _ := config.LoadDefaultConfig(context.TODO())
svc := s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = aws.String(os.Getenv("AWS_ENDPOINT_URL_S3"))
})
// retrieve accounts from db
_, err := svc.CreateBucket(context.Background(), &s3.CreateBucketInput{
Bucket: aws.String("pokerogue-system"),
})
if err != nil {
log.Printf("error while creating bucket: %s", err)
}
accounts := db.RetrieveOldAccounts()
for _, user := range accounts {
data, _ := db.ReadSystemSaveData(user)
username, _ := db.FetchUsernameFromUUID(user)
json, _ := json.Marshal(data)
_, err := svc.PutObject(context.Background(), &s3.PutObjectInput{
Bucket: aws.String("pokerogue-system"),
Key: aws.String(username),
Body: bytes.NewReader(json),
})
if err != nil {
log.Printf("error while saving data in s3 for user %s: %s", username, err)
continue
}
fmt.Printf("Saved data in s3 for user %s\n", username)
db.UpdateLocation(user, username)
}
}

@ -420,6 +420,7 @@ func handleSystem(w http.ResponseWriter, r *http.Request) {
if errors.Is(err, sql.ErrNoRows) {
http.Error(w, err.Error(), http.StatusNotFound)
} else {
fmt.Printf("failed to get system save data: %s\n", err)
httpError(w, r, err, http.StatusInternalServerError)
}

@ -103,6 +103,11 @@ func setupDb(tx *sql.Tx) error {
`ALTER TABLE accounts ADD COLUMN IF NOT EXISTS discordId VARCHAR(32) UNIQUE DEFAULT NULL`,
`ALTER TABLE accounts ADD COLUMN IF NOT EXISTS googleId VARCHAR(32) UNIQUE DEFAULT NULL`,
// ----------------------------------
// MIGRATION 004
`ALTER TABLE accounts ADD COLUMN IF NOT EXISTS isInLocalDb TINYINT(1) NOT NULL DEFAULT 1`,
}
for _, q := range queries {

@ -19,10 +19,17 @@ package db
import (
"bytes"
"context"
"encoding/gob"
"encoding/json"
"fmt"
"github.com/klauspost/compress/zstd"
"github.com/pagefaultgames/rogueserver/defs"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
func TryAddSeedCompletion(uuid []byte, seed string, mode int) (bool, error) {
@ -55,8 +62,16 @@ func ReadSeedCompleted(uuid []byte, seed string) (bool, error) {
func ReadSystemSaveData(uuid []byte) (defs.SystemSaveData, error) {
var system defs.SystemSaveData
isLocal, err := isSaveInLocalDb(uuid)
if err != nil {
return system, err
}
if !isLocal {
RetrieveSystemSaveFromS3(uuid)
}
var data []byte
err := handle.QueryRow("SELECT data FROM systemSaveData WHERE uuid = ?", uuid).Scan(&data)
err = handle.QueryRow("SELECT data FROM systemSaveData WHERE uuid = ?", uuid).Scan(&data)
if err != nil {
return system, err
}
@ -193,3 +208,95 @@ func RetrievePlaytime(uuid []byte) (int, error) {
return playtime, nil
}
func isSaveInLocalDb(uuid []byte) (bool, error) {
var isLocal bool
err := handle.QueryRow("SELECT isInLocalDb FROM accounts WHERE uuid = ?", uuid).Scan(&isLocal)
if err != nil {
return false, err
}
return isLocal, nil
}
func RetrieveSystemSaveFromS3(uuid []byte) error {
cfg, err := config.LoadDefaultConfig(context.TODO())
if err != nil {
return err
}
client := s3.NewFromConfig(cfg)
username, err := FetchUsernameFromUUID(uuid)
if err != nil {
return err
}
s3Object := &s3.GetObjectInput{
Bucket: aws.String("pokerogue-system"),
Key: aws.String(username),
}
resp, err := client.GetObject(context.TODO(), s3Object)
if err != nil {
return err
}
var session defs.SystemSaveData
json.NewDecoder(resp.Body).Decode(&session)
err = StoreSystemSaveData(uuid, session)
if err != nil {
fmt.Printf("Failed to store system save data from s3 for user %s\n", username)
return err
}
fmt.Printf("Retrieved system save data from s3 for user %s\n", username)
_, err = handle.Exec("UPDATE accounts SET isInLocalDb = 1 WHERE uuid = ?", uuid)
if err != nil {
return err
}
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String("pokerogue-system"),
Key: aws.String(username),
})
if err != nil {
fmt.Printf("Failed to delete object %s from s3: %s\n", username, err)
}
return nil
}
func RetrieveOldAccounts() [][]byte {
var users [][]byte
rows, err := handle.Query("SELECT uuid FROM accounts WHERE isInLocalDb = 1 && lastActivity < DATE_SUB(NOW(), INTERVAL 3 MONTH) LIMIT 3000")
if err != nil {
return nil
}
defer rows.Close()
for rows.Next() {
var uuid []byte
if err := rows.Scan(&uuid); err != nil {
return nil
}
users = append(users, uuid)
}
if err := rows.Err(); err != nil {
return nil
}
return users
}
func UpdateLocation(uuid []byte, username string) {
_, err := handle.Exec("UPDATE accounts SET isInLocalDb = 0 WHERE uuid = ?", uuid)
if err != nil {
fmt.Printf("Failed to update location for user %s\n", username)
return
}
DeleteSystemSaveData(uuid)
}

@ -1,6 +1,6 @@
services:
server:
image: ghcr.io/pagefaultgames/rogueserver:master
image: rogueserver:latest
restart: unless-stopped
environment:
debug: true
@ -10,6 +10,11 @@ services:
dbname: pokeroguedb
gameurl: http://localhost:8000
callbackurl: http://localhost:8001
AWS_ACCESS_KEY_ID: <access>
AWS_SECRET_ACCESS_KEY: <secret>
AWS_REGION: <region>
AWS_ENDPOINT_URL_S3: <endpoint>
depends_on:
db:
condition: service_healthy

@ -9,12 +9,30 @@ require (
)
require (
github.com/aws/aws-sdk-go-v2 v1.32.2
github.com/aws/aws-sdk-go-v2/config v1.27.43
github.com/aws/aws-sdk-go-v2/service/s3 v1.65.0
github.com/bwmarrin/discordgo v0.28.1
github.com/golang-jwt/jwt/v5 v5.2.1
github.com/klauspost/compress v1.17.9
)
require (
github.com/bwmarrin/discordgo v0.28.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.41 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.19 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect
github.com/aws/smithy-go v1.22.0 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
golang.org/x/sys v0.19.0 // indirect
)

@ -1,13 +1,49 @@
github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI=
github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA=
github.com/aws/aws-sdk-go-v2/config v1.27.43 h1:p33fDDihFC390dhhuv8nOmX419wjOSDQRb+USt20RrU=
github.com/aws/aws-sdk-go-v2/config v1.27.43/go.mod h1:pYhbtvg1siOOg8h5an77rXle9tVG8T+BWLWAo7cOukc=
github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0gizGINCOWxSLY9W8=
github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 h1:TMH3f/SCAWdNtXXVPPu5D6wrr4G5hI1rAxbcocKfC7Q=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17/go.mod h1:1ZRXLdTpzdJb9fwTMXiLipENRxkGMTn1sfKexGllQCw=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79WzpG/ixT9FZvQlh5HY1NRqSHBNOCk=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.19 h1:FKdiFzTxlTRO71p0C7VrLbkkdW8qfMKF5+ej6bTmkT0=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.19/go.mod h1:abO3pCj7WLQPTllnSeYImqFfkGrmJV0JovWo/gqT5N0=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.0 h1:FQNWhRuSq8QwW74GtU0MrveNhZbqvHsA4dkA9w8fTDQ=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.0/go.mod h1:j/zZ3zmWfGCK91K73YsfHP53BSTLSjL/y6YN39XbBLM=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 h1:s7NA1SOw8q/5c0wr8477yOPp0z+uBaXBnLE0XYb0POA=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2/go.mod h1:fnjjWyAW/Pj5HYOxl9LJqWtEwS7W2qgcRLWP+uWbss0=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.0 h1:1NKXS8XfhMM0bg5wVYa/eOH8AM2f6JijugbKEyQFTIg=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.0/go.mod h1:ph931DUfVfgrhZR7py9olSvHCiRpvaGxNvlWBcXxFds=
github.com/aws/aws-sdk-go-v2/service/s3 v1.65.0 h1:2dSm7frMrw2tdJ0QvyccQNJyPGaP24dyDgZ6h1QJMGU=
github.com/aws/aws-sdk-go-v2/service/s3 v1.65.0/go.mod h1:4XSVpw66upN8wND3JZA29eXl2NOZvfFVq7DIP6xvfuQ=
github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 h1:bSYXVyUzoTHoKalBmwaZxs97HU9DWWI3ehHSAMa7xOk=
github.com/aws/aws-sdk-go-v2/service/sso v1.24.2/go.mod h1:skMqY7JElusiOUjMJMOv1jJsP7YUg7DrhgqZZWuzu1U=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 h1:AhmO1fHINP9vFYUE0LHzCWg/LfUWUF+zFPEcY9QXb7o=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2/go.mod h1:o8aQygT2+MVP0NaV6kbdE1YnnIM8RRVQzoeUH45GOdI=
github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 h1:CiS7i0+FUe+/YY1GvIBLLrR/XNGZ4CtM1Ll0XavNuVo=
github.com/aws/aws-sdk-go-v2/service/sts v1.32.2/go.mod h1:HtaiBI8CjYoNVde8arShXb94UbQQi9L4EMr6D+xGBwo=
github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM=
github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
github.com/bwmarrin/discordgo v0.28.1 h1:gXsuo2GBO7NbR6uqmrrBDplPUx2T3nzu775q/Rd1aG4=
github.com/bwmarrin/discordgo v0.28.1/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY=
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=

Loading…
Cancel
Save