backup: Add backup capabilities using an S3-compatible API.
This commit is contained in:
parent
b8ae6e364a
commit
c62b542552
9 changed files with 198 additions and 4 deletions
23
.github/workflows/main.yml
vendored
23
.github/workflows/main.yml
vendored
|
@ -7,7 +7,28 @@ on:
|
||||||
- '**'
|
- '**'
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build-and-test:
|
build-backup-job:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Login to DockerHub
|
||||||
|
uses: docker/login-action@v2
|
||||||
|
with:
|
||||||
|
username: ${{ secrets.DOCKER_USERNAME }}
|
||||||
|
password: ${{ secrets.DOCKER_TOKEN }}
|
||||||
|
|
||||||
|
- name: Build/push Docker image (main)
|
||||||
|
if: github.ref == 'refs/heads/main'
|
||||||
|
uses: docker/build-push-action@v6
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: build/guestbooky-backup/Dockerfile
|
||||||
|
push: true
|
||||||
|
tags: ${{ secrets.DOCKER_USERNAME }}/guestbooky-backup:latest
|
||||||
|
|
||||||
|
build-and-test-backend:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
|
|
|
@ -10,3 +10,9 @@ MONGO_INITDB_ROOT_PASSWORD=mongo
|
||||||
GUESTBOOKY_DB_NAME=Guestbooky
|
GUESTBOOKY_DB_NAME=Guestbooky
|
||||||
GUESTBOOKY_USER=guestbookyuser
|
GUESTBOOKY_USER=guestbookyuser
|
||||||
GUESTBOOKY_PASSWORD=guestbookypassword
|
GUESTBOOKY_PASSWORD=guestbookypassword
|
||||||
|
BACKUP_BASE_PATH=/backups
|
||||||
|
BACKUP_S3_KEY_NAME=guestbooky-backup
|
||||||
|
BACKUP_S3_ACCESS_ID=0000000000000000000000000
|
||||||
|
BACKUP_S3_SECRET_ID=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
|
||||||
|
BACKUP_S3_ENDPOINT=https://s3.us-east-005.backblazeb2.com
|
||||||
|
BACKUP_S3_REGION=us-east-005
|
|
@ -50,6 +50,7 @@ services:
|
||||||
- ./docker-compose.yml:/etc/docker-compose.yml:ro
|
- ./docker-compose.yml:/etc/docker-compose.yml:ro
|
||||||
labels:
|
labels:
|
||||||
ofelia.job-run.backup: "0 0 * * * docker-compose -f /etc/docker-compose.yml run --rm backup-job"
|
ofelia.job-run.backup: "0 0 * * * docker-compose -f /etc/docker-compose.yml run --rm backup-job"
|
||||||
|
ofelia.job-run.upload: "0 0 * * * docker-compose -f /etc/docker-compose.yml run --rm upload"
|
||||||
networks:
|
networks:
|
||||||
- guestbooky
|
- guestbooky
|
||||||
env_file:
|
env_file:
|
||||||
|
@ -65,13 +66,32 @@ services:
|
||||||
--username ${GUESTBOOKY_USER}
|
--username ${GUESTBOOKY_USER}
|
||||||
--password ${GUESTBOOKY_USER}
|
--password ${GUESTBOOKY_USER}
|
||||||
--authenticationDatabase ${GUESTBOOKY_DB_NAME}
|
--authenticationDatabase ${GUESTBOOKY_DB_NAME}
|
||||||
--out /backups/guestbooky_$(date +\%Y-\%m-\%d)"
|
--out /backups/guestbooky_$(date +\%Y-\%m-\%d) && touch /backups/backup_done"
|
||||||
volumes:
|
volumes:
|
||||||
- ./backups:/backups
|
- ./backups:/backups
|
||||||
depends_on:
|
depends_on:
|
||||||
- mongo
|
- mongo
|
||||||
networks:
|
networks:
|
||||||
- guestbooky
|
- guestbooky
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "test", "-f", "/backups/backup_done"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 10s
|
||||||
|
retries: 3
|
||||||
|
|
||||||
|
upload:
|
||||||
|
image: cotti/guestbooky-backup
|
||||||
|
container_name: guestbooky-backup
|
||||||
|
environment:
|
||||||
|
BACKUP_S3_KEY_NAME: ${BACKUP_S3_KEY_NAME}
|
||||||
|
BACKUP_S3_ACCESS_ID: ${BACKUP_S3_ACCESS_ID}
|
||||||
|
BACKUP_S3_SECRET_ID: ${BACKUP_S3_SECRET_ID}
|
||||||
|
BACKUP_S3_ENDPOINT: ${BACKUP_S3_ENDPOINT}
|
||||||
|
BACKUP_S3_REGION: ${BACKUP_S3_REGION}
|
||||||
|
BACKUP_SOURCE_PATH: ${BACKUP_SOURCE_PATH}
|
||||||
|
BACKUP_DESTINATION_PATH: ${BACKUP_DESTINATION_PATH}.gzip
|
||||||
|
volumes:
|
||||||
|
- ./backups:/backups
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
mongodata:
|
mongodata:
|
||||||
|
|
21
build/guestbooky-backup/Dockerfile
Normal file
21
build/guestbooky-backup/Dockerfile
Normal file
|
@ -0,0 +1,21 @@
|
||||||
|
FROM golang:1.23.1 AS builder
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
ENV GO111MODULE=on
|
||||||
|
|
||||||
|
COPY ../../src/Guestbooky-backup/ .
|
||||||
|
|
||||||
|
RUN go mod download
|
||||||
|
|
||||||
|
RUN go build -o guestbooky-backup .
|
||||||
|
|
||||||
|
# Start a new stage from scratch
|
||||||
|
FROM alpine:latest
|
||||||
|
|
||||||
|
WORKDIR /root/
|
||||||
|
|
||||||
|
# Copy the Pre-built binary file from the previous stage
|
||||||
|
COPY --from=builder /app/guestbooky-backup .
|
||||||
|
|
||||||
|
# Command to run the executable
|
||||||
|
CMD ["./guestbooky-backup"]
|
|
@ -5,13 +5,29 @@ import (
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
"github.com/cotti/Guestbooky-backup/internal/compactor"
|
"github.com/cotti/Guestbooky-backup/internal/compactor"
|
||||||
|
"github.com/cotti/Guestbooky-backup/internal/uploader"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
err := compactor.Compact(os.Args[1], os.Args[2])
|
sourceFile := os.Getenv("BACKUP_SOURCE_PATH")
|
||||||
|
destinationFile := os.Getenv("BACKUP_DESTINATION_PATH")
|
||||||
|
err := compactor.Compact(sourceFile, destinationFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Println("An error occurred while compacting:", err.Error())
|
fmt.Println("An error occurred while compacting:", err.Error())
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
err = uploader.Upload(destinationFile)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println("An error occurred while uploading:", err.Error())
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = os.Remove("/backups/backups_done")
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println("An error occurred while removing the file:", err.Error())
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
os.Exit(0)
|
os.Exit(0)
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,3 +1,8 @@
|
||||||
module github.com/cotti/Guestbooky-backup
|
module github.com/cotti/Guestbooky-backup
|
||||||
|
|
||||||
go 1.23.1
|
go 1.23.1
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/aws/aws-sdk-go v1.55.6 // indirect
|
||||||
|
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||||
|
)
|
||||||
|
|
10
src/Guestbooky-backup/go.sum
Normal file
10
src/Guestbooky-backup/go.sum
Normal file
|
@ -0,0 +1,10 @@
|
||||||
|
github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk=
|
||||||
|
github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
|
||||||
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||||
|
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||||
|
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
|
@ -1,8 +1,10 @@
|
||||||
package compactor
|
package compactor
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"compress/gzip"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
)
|
)
|
||||||
|
@ -16,5 +18,24 @@ func Compact(source, destination string) error {
|
||||||
return errors.New("source file does not exist")
|
return errors.New("source file does not exist")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
originFileHandle, err := os.Open(source)
|
||||||
|
if err != nil {
|
||||||
|
return errors.New("failed to open source file")
|
||||||
|
}
|
||||||
|
defer originFileHandle.Close()
|
||||||
|
|
||||||
|
destinationFileHandle, err := os.Create(destination)
|
||||||
|
if err != nil {
|
||||||
|
return errors.New("failed to create destination file")
|
||||||
|
}
|
||||||
|
defer destinationFileHandle.Close()
|
||||||
|
|
||||||
|
zipWriter := gzip.NewWriter(destinationFileHandle)
|
||||||
|
defer zipWriter.Close()
|
||||||
|
|
||||||
|
if _, err := io.Copy(zipWriter, originFileHandle); err != nil {
|
||||||
|
return errors.New("failed to copy file")
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
74
src/Guestbooky-backup/internal/uploader/uploader.go
Normal file
74
src/Guestbooky-backup/internal/uploader/uploader.go
Normal file
|
@ -0,0 +1,74 @@
|
||||||
|
package uploader
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/aws/aws-sdk-go/aws"
|
||||||
|
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||||
|
"github.com/aws/aws-sdk-go/aws/session"
|
||||||
|
"github.com/aws/aws-sdk-go/service/s3"
|
||||||
|
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||||
|
)
|
||||||
|
|
||||||
|
type S3Client struct {
|
||||||
|
bucketName string
|
||||||
|
s3Client *s3.S3
|
||||||
|
}
|
||||||
|
|
||||||
|
func Upload(compactedFile string) error {
|
||||||
|
fmt.Println(
|
||||||
|
"Uploading", compactedFile, "to storage",
|
||||||
|
)
|
||||||
|
|
||||||
|
s3Client, err := createS3Client()
|
||||||
|
if err != nil {
|
||||||
|
return errors.New("failed to create S3 client")
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := os.Open(compactedFile)
|
||||||
|
if err != nil {
|
||||||
|
return errors.New("failed to open file")
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
uploader := s3manager.NewUploaderWithClient(s3Client.s3Client)
|
||||||
|
|
||||||
|
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||||
|
Bucket: &s3Client.bucketName,
|
||||||
|
Key: &compactedFile,
|
||||||
|
Body: file,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return errors.New("failed to upload file")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func createS3Client() (*S3Client, error) {
|
||||||
|
keyId := os.Getenv("BACKUP_S3_ACCESS_ID")
|
||||||
|
applicationKey := os.Getenv("BACKUP_S3_SECRET_ID")
|
||||||
|
bucketName := os.Getenv("BACKUP_S3_KEY_NAME")
|
||||||
|
endpoint := os.Getenv("BACKUP_S3_ENDPOINT")
|
||||||
|
region := os.Getenv("BACKUP_S3_REGION")
|
||||||
|
|
||||||
|
s3Config := &aws.Config{
|
||||||
|
Credentials: credentials.NewStaticCredentials(keyId, applicationKey, ""),
|
||||||
|
Endpoint: &endpoint,
|
||||||
|
Region: ®ion,
|
||||||
|
}
|
||||||
|
|
||||||
|
awsSession, err := session.NewSession(s3Config)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.New("failed to create S3 session")
|
||||||
|
}
|
||||||
|
|
||||||
|
s3Client := s3.New(awsSession)
|
||||||
|
|
||||||
|
return &S3Client{
|
||||||
|
bucketName: bucketName,
|
||||||
|
s3Client: s3Client,
|
||||||
|
}, nil
|
||||||
|
}
|
Loading…
Reference in a new issue