From b8ae6e364ad97379ee8b1c3a0206e3281f6fe466 Mon Sep 17 00:00:00 2001 From: Felipe Cotti Date: Fri, 6 Dec 2024 18:39:47 -0300 Subject: [PATCH 1/2] Starting implementation of the backup service. --- .../guestbooky-backup/guestbooky-backup.go | 17 ++++++++++++++++ src/Guestbooky-backup/go.mod | 3 +++ .../internal/compactor/compactor.go | 20 +++++++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 src/Guestbooky-backup/cmd/guestbooky-backup/guestbooky-backup.go create mode 100644 src/Guestbooky-backup/go.mod create mode 100644 src/Guestbooky-backup/internal/compactor/compactor.go diff --git a/src/Guestbooky-backup/cmd/guestbooky-backup/guestbooky-backup.go b/src/Guestbooky-backup/cmd/guestbooky-backup/guestbooky-backup.go new file mode 100644 index 0000000..864b7da --- /dev/null +++ b/src/Guestbooky-backup/cmd/guestbooky-backup/guestbooky-backup.go @@ -0,0 +1,17 @@ +package main + +import ( + "fmt" + "os" + + "github.com/cotti/Guestbooky-backup/internal/compactor" +) + +func main() { + err := compactor.Compact(os.Args[1], os.Args[2]) + if err != nil { + fmt.Println("An error occurred while compacting:", err.Error()) + os.Exit(1) + } + os.Exit(0) +} diff --git a/src/Guestbooky-backup/go.mod b/src/Guestbooky-backup/go.mod new file mode 100644 index 0000000..8084f1f --- /dev/null +++ b/src/Guestbooky-backup/go.mod @@ -0,0 +1,3 @@ +module github.com/cotti/Guestbooky-backup + +go 1.23.1 diff --git a/src/Guestbooky-backup/internal/compactor/compactor.go b/src/Guestbooky-backup/internal/compactor/compactor.go new file mode 100644 index 0000000..ed662e7 --- /dev/null +++ b/src/Guestbooky-backup/internal/compactor/compactor.go @@ -0,0 +1,20 @@ +package compactor + +import ( + "errors" + "fmt" + "io/fs" + "os" +) + +func Compact(source, destination string) error { + fmt.Println( + "Compacting", source, "to", destination, + ) + + if _, err := os.Stat(source); errors.Is(err, fs.ErrNotExist) { + return errors.New("source file does not exist") + } + + return nil +} -- 2.45.2 From c62b5425521100a5b33dd9806c3ba7aea66e76e7 Mon Sep 17 00:00:00 2001 From: Felipe Cotti Date: Mon, 27 Jan 2025 01:24:15 -0300 Subject: [PATCH 2/2] backup: Add backup capabilities using an S3-compatible API. --- .github/workflows/main.yml | 23 +++++- build/.env.template | 8 +- build/docker-compose.public.yml | 22 +++++- build/guestbooky-backup/Dockerfile | 21 ++++++ .../guestbooky-backup/guestbooky-backup.go | 18 ++++- src/Guestbooky-backup/go.mod | 5 ++ src/Guestbooky-backup/go.sum | 10 +++ .../internal/compactor/compactor.go | 21 ++++++ .../internal/uploader/uploader.go | 74 +++++++++++++++++++ 9 files changed, 198 insertions(+), 4 deletions(-) create mode 100644 build/guestbooky-backup/Dockerfile create mode 100644 src/Guestbooky-backup/go.sum create mode 100644 src/Guestbooky-backup/internal/uploader/uploader.go diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e254914..3cc3fa5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,7 +7,28 @@ on: - '**' jobs: - build-and-test: + build-backup-job: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build/push Docker image (main) + if: github.ref == 'refs/heads/main' + uses: docker/build-push-action@v6 + with: + context: . + file: build/guestbooky-backup/Dockerfile + push: true + tags: ${{ secrets.DOCKER_USERNAME }}/guestbooky-backup:latest + + build-and-test-backend: runs-on: ubuntu-latest steps: - name: Checkout diff --git a/build/.env.template b/build/.env.template index 555e1ea..1b10862 100644 --- a/build/.env.template +++ b/build/.env.template @@ -9,4 +9,10 @@ MONGO_INITDB_ROOT_USERNAME=root MONGO_INITDB_ROOT_PASSWORD=mongo GUESTBOOKY_DB_NAME=Guestbooky GUESTBOOKY_USER=guestbookyuser -GUESTBOOKY_PASSWORD=guestbookypassword \ No newline at end of file +GUESTBOOKY_PASSWORD=guestbookypassword +BACKUP_BASE_PATH=/backups +BACKUP_S3_KEY_NAME=guestbooky-backup +BACKUP_S3_ACCESS_ID=0000000000000000000000000 +BACKUP_S3_SECRET_ID=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +BACKUP_S3_ENDPOINT=https://s3.us-east-005.backblazeb2.com +BACKUP_S3_REGION=us-east-005 \ No newline at end of file diff --git a/build/docker-compose.public.yml b/build/docker-compose.public.yml index dad7463..d8d614b 100644 --- a/build/docker-compose.public.yml +++ b/build/docker-compose.public.yml @@ -50,6 +50,7 @@ services: - ./docker-compose.yml:/etc/docker-compose.yml:ro labels: ofelia.job-run.backup: "0 0 * * * docker-compose -f /etc/docker-compose.yml run --rm backup-job" + ofelia.job-run.upload: "0 0 * * * docker-compose -f /etc/docker-compose.yml run --rm upload" networks: - guestbooky env_file: @@ -65,13 +66,32 @@ services: --username ${GUESTBOOKY_USER} --password ${GUESTBOOKY_USER} --authenticationDatabase ${GUESTBOOKY_DB_NAME} - --out /backups/guestbooky_$(date +\%Y-\%m-\%d)" + --out /backups/guestbooky_$(date +\%Y-\%m-\%d) && touch /backups/backup_done" volumes: - ./backups:/backups depends_on: - mongo networks: - guestbooky + healthcheck: + test: ["CMD", "test", "-f", "/backups/backup_done"] + interval: 30s + timeout: 10s + retries: 3 + + upload: + image: cotti/guestbooky-backup + container_name: guestbooky-backup + environment: + BACKUP_S3_KEY_NAME: ${BACKUP_S3_KEY_NAME} + BACKUP_S3_ACCESS_ID: ${BACKUP_S3_ACCESS_ID} + BACKUP_S3_SECRET_ID: ${BACKUP_S3_SECRET_ID} + BACKUP_S3_ENDPOINT: ${BACKUP_S3_ENDPOINT} + BACKUP_S3_REGION: ${BACKUP_S3_REGION} + BACKUP_SOURCE_PATH: ${BACKUP_SOURCE_PATH} + BACKUP_DESTINATION_PATH: ${BACKUP_DESTINATION_PATH}.gzip + volumes: + - ./backups:/backups volumes: mongodata: diff --git a/build/guestbooky-backup/Dockerfile b/build/guestbooky-backup/Dockerfile new file mode 100644 index 0000000..0bde9b1 --- /dev/null +++ b/build/guestbooky-backup/Dockerfile @@ -0,0 +1,21 @@ +FROM golang:1.23.1 AS builder + +WORKDIR /app +ENV GO111MODULE=on + +COPY ../../src/Guestbooky-backup/ . + +RUN go mod download + +RUN go build -o guestbooky-backup . + +# Start a new stage from scratch +FROM alpine:latest + +WORKDIR /root/ + +# Copy the Pre-built binary file from the previous stage +COPY --from=builder /app/guestbooky-backup . + +# Command to run the executable +CMD ["./guestbooky-backup"] \ No newline at end of file diff --git a/src/Guestbooky-backup/cmd/guestbooky-backup/guestbooky-backup.go b/src/Guestbooky-backup/cmd/guestbooky-backup/guestbooky-backup.go index 864b7da..59fbf86 100644 --- a/src/Guestbooky-backup/cmd/guestbooky-backup/guestbooky-backup.go +++ b/src/Guestbooky-backup/cmd/guestbooky-backup/guestbooky-backup.go @@ -5,13 +5,29 @@ import ( "os" "github.com/cotti/Guestbooky-backup/internal/compactor" + "github.com/cotti/Guestbooky-backup/internal/uploader" ) func main() { - err := compactor.Compact(os.Args[1], os.Args[2]) + sourceFile := os.Getenv("BACKUP_SOURCE_PATH") + destinationFile := os.Getenv("BACKUP_DESTINATION_PATH") + err := compactor.Compact(sourceFile, destinationFile) if err != nil { fmt.Println("An error occurred while compacting:", err.Error()) os.Exit(1) } + + err = uploader.Upload(destinationFile) + if err != nil { + fmt.Println("An error occurred while uploading:", err.Error()) + os.Exit(1) + } + + err = os.Remove("/backups/backups_done") + if err != nil { + fmt.Println("An error occurred while removing the file:", err.Error()) + os.Exit(1) + } + os.Exit(0) } diff --git a/src/Guestbooky-backup/go.mod b/src/Guestbooky-backup/go.mod index 8084f1f..812726a 100644 --- a/src/Guestbooky-backup/go.mod +++ b/src/Guestbooky-backup/go.mod @@ -1,3 +1,8 @@ module github.com/cotti/Guestbooky-backup go 1.23.1 + +require ( + github.com/aws/aws-sdk-go v1.55.6 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect +) diff --git a/src/Guestbooky-backup/go.sum b/src/Guestbooky-backup/go.sum new file mode 100644 index 0000000..f145b0f --- /dev/null +++ b/src/Guestbooky-backup/go.sum @@ -0,0 +1,10 @@ +github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk= +github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/src/Guestbooky-backup/internal/compactor/compactor.go b/src/Guestbooky-backup/internal/compactor/compactor.go index ed662e7..5db45c7 100644 --- a/src/Guestbooky-backup/internal/compactor/compactor.go +++ b/src/Guestbooky-backup/internal/compactor/compactor.go @@ -1,8 +1,10 @@ package compactor import ( + "compress/gzip" "errors" "fmt" + "io" "io/fs" "os" ) @@ -16,5 +18,24 @@ func Compact(source, destination string) error { return errors.New("source file does not exist") } + originFileHandle, err := os.Open(source) + if err != nil { + return errors.New("failed to open source file") + } + defer originFileHandle.Close() + + destinationFileHandle, err := os.Create(destination) + if err != nil { + return errors.New("failed to create destination file") + } + defer destinationFileHandle.Close() + + zipWriter := gzip.NewWriter(destinationFileHandle) + defer zipWriter.Close() + + if _, err := io.Copy(zipWriter, originFileHandle); err != nil { + return errors.New("failed to copy file") + } + return nil } diff --git a/src/Guestbooky-backup/internal/uploader/uploader.go b/src/Guestbooky-backup/internal/uploader/uploader.go new file mode 100644 index 0000000..99b368a --- /dev/null +++ b/src/Guestbooky-backup/internal/uploader/uploader.go @@ -0,0 +1,74 @@ +package uploader + +import ( + "errors" + "fmt" + "os" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" +) + +type S3Client struct { + bucketName string + s3Client *s3.S3 +} + +func Upload(compactedFile string) error { + fmt.Println( + "Uploading", compactedFile, "to storage", + ) + + s3Client, err := createS3Client() + if err != nil { + return errors.New("failed to create S3 client") + } + + file, err := os.Open(compactedFile) + if err != nil { + return errors.New("failed to open file") + } + defer file.Close() + + uploader := s3manager.NewUploaderWithClient(s3Client.s3Client) + + _, err = uploader.Upload(&s3manager.UploadInput{ + Bucket: &s3Client.bucketName, + Key: &compactedFile, + Body: file, + }) + if err != nil { + return errors.New("failed to upload file") + } + + return nil +} + +func createS3Client() (*S3Client, error) { + keyId := os.Getenv("BACKUP_S3_ACCESS_ID") + applicationKey := os.Getenv("BACKUP_S3_SECRET_ID") + bucketName := os.Getenv("BACKUP_S3_KEY_NAME") + endpoint := os.Getenv("BACKUP_S3_ENDPOINT") + region := os.Getenv("BACKUP_S3_REGION") + + s3Config := &aws.Config{ + Credentials: credentials.NewStaticCredentials(keyId, applicationKey, ""), + Endpoint: &endpoint, + Region: ®ion, + } + + awsSession, err := session.NewSession(s3Config) + if err != nil { + return nil, errors.New("failed to create S3 session") + } + + s3Client := s3.New(awsSession) + + return &S3Client{ + bucketName: bucketName, + s3Client: s3Client, + }, nil +} -- 2.45.2