goc_marek/zfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

simplyfy

Browse Source
This commit is contained in:
Marek Goc
2026-02-14 19:09:05 +01:00
parent 05c916e9a9
commit 5892ac2a2e
13 changed files with 394 additions and 721 deletions
Download Patch File Download Diff File Expand all files Collapse all files

246
internal/client/client.go
View File

@@ -4,6 +4,7 @@ package client
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
@@ -13,6 +14,10 @@ import (
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/mistifyio/go-zfs"
"github.com/pierrec/lz4/v4"
)
@@ -109,10 +114,40 @@ func (c *Client) SendSnapshot(snapshot *zfs.Dataset) error {
return c.sendViaZFS(snapshot, uploadResp.StorageKey)
}
// streamToS3 streams a ZFS snapshot to S3 storage via HTTP.
// streamToS3 streams a ZFS snapshot to S3 storage using AWS SDK.
// The snapshot is optionally compressed with LZ4 before transmission.
func (c *Client) streamToS3(snapshot *zfs.Dataset, uploadURL, storageKey string) error {
fmt.Printf("→ Streaming snapshot to S3...\n")
fmt.Printf("→ Uploading snapshot to S3...\n")
// Ensure endpoint has valid URI scheme
endpoint := c.config.S3Endpoint
if endpoint != "" && !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") {
endpoint = "http://" + endpoint
}
// Create AWS config
awsCfg, err := config.LoadDefaultConfig(context.TODO(),
config.WithRegion(c.config.S3Region),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
c.config.S3AccessKey,
c.config.S3SecretKey,
"",
)),
)
if err != nil {
return fmt.Errorf("failed to load AWS config: %v", err)
}
// Determine if using custom endpoint (non-AWS)
customEndpoint := endpoint != "" && endpoint != "http://s3.amazonaws.com" && endpoint != "https://s3.amazonaws.com"
// Create S3 client
s3Client := s3.NewFromConfig(awsCfg, func(o *s3.Options) {
if customEndpoint {
o.BaseEndpoint = aws.String(endpoint)
o.UsePathStyle = true // Required for MinIO compatible storage
}
})
// Create ZFS send command
cmd := exec.Command("zfs", "send", snapshot.Name)
@@ -145,57 +180,24 @@ func (c *Client) streamToS3(snapshot *zfs.Dataset, uploadURL, storageKey string)
reader = pr
}
// Create HTTP request
req, err := http.NewRequest("POST", c.config.ServerURL+uploadURL, reader)
if err != nil {
return fmt.Errorf("failed to create request: %v", err)
}
// Set required headers
req.Header.Set("X-API-Key", c.config.APIKey)
req.Header.Set("X-Storage-Key", storageKey)
req.Header.Set("X-Dataset-Name", c.config.LocalDataset)
req.Header.Set("X-Compressed", fmt.Sprintf("%v", c.config.Compress))
req.Header.Set("Content-Type", "application/octet-stream")
// Send request with no timeout for large uploads
client := &http.Client{
Timeout: 0,
}
httpResp, err := client.Do(req)
if err != nil {
cmd.Process.Kill()
return fmt.Errorf("failed to upload: %v", err)
}
defer httpResp.Body.Close()
if httpResp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(httpResp.Body)
return fmt.Errorf("upload failed with status %d: %s", httpResp.StatusCode, body)
}
// Upload to S3 using PutObject
_, err = s3Client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(c.config.S3Bucket),
Key: aws.String(storageKey),
Body: reader,
ContentType: aws.String("application/octet-stream"),
})
// Wait for zfs send to complete
if err := cmd.Wait(); err != nil {
return fmt.Errorf("zfs send failed: %v", err)
}
// Parse response
var result struct {
Success bool `json:"success"`
Message string `json:"message"`
Size int64 `json:"size"`
if err != nil {
return fmt.Errorf("failed to upload to S3: %v", err)
}
if err := json.NewDecoder(httpResp.Body).Decode(&result); err != nil {
return fmt.Errorf("failed to decode response: %v", err)
}
if !result.Success {
return fmt.Errorf("upload failed: %s", result.Message)
}
fmt.Printf("✓ Snapshot uploaded successfully!\n")
fmt.Printf(" Size: %.2f MB\n", float64(result.Size)/(1024*1024))
fmt.Printf("✓ Snapshot uploaded to S3 successfully!\n")
return nil
}
@@ -234,6 +236,55 @@ func (c *Client) sendViaZFS(snapshot *zfs.Dataset, receivePath string) error {
return nil
}
// SnapshotResult contains the result of a snapshot creation and send operation.
type SnapshotResult struct {
FullBackup bool
Snapshot *zfs.Dataset
}
// CreateAndSend creates a snapshot and sends it to the backup server.
// It automatically detects if this is a full or incremental backup:
// - If no bookmark exists, does a full backup
// - If bookmark exists, does an incremental backup from the bookmark
func (c *Client) CreateAndSend() (*SnapshotResult, error) {
// Check for existing bookmark to determine backup type
lastBookmark, err := c.GetLastBookmark()
if err != nil {
return nil, fmt.Errorf("failed to check bookmarks: %v", err)
}
// Create new snapshot
snapshot, err := c.CreateSnapshot()
if err != nil {
return nil, fmt.Errorf("failed to create snapshot: %v", err)
}
isFullBackup := lastBookmark == ""
if isFullBackup {
fmt.Println("→ No previous backup found, doing FULL backup...")
// Send as full (no base)
if err := c.SendIncremental(snapshot, ""); err != nil {
return nil, fmt.Errorf("failed to send snapshot: %v", err)
}
} else {
fmt.Printf("→ Found previous backup, doing INCREMENTAL from %s...", lastBookmark)
// Send as incremental from bookmark
if err := c.SendIncremental(snapshot, lastBookmark); err != nil {
return nil, fmt.Errorf("failed to send incremental: %v", err)
}
}
// Create bookmark for future incremental backups
if err := c.CreateBookmark(snapshot); err != nil {
fmt.Printf("Warning: failed to create bookmark: %v\n", err)
}
return &SnapshotResult{
FullBackup: isFullBackup,
Snapshot: snapshot,
}, nil
}
// GetStatus retrieves and displays the client's backup status from the server.
// Shows storage usage, quota, and snapshot count.
func (c *Client) GetStatus() error {
@@ -273,106 +324,3 @@ func (c *Client) GetStatus() error {
return nil
}
// RequestRotation asks the server to rotate old snapshots.
// This deletes the oldest snapshots to free up space.
func (c *Client) RequestRotation() error {
reqBody, _ := json.Marshal(map[string]string{
"client_id": c.config.ClientID,
"api_key": c.config.APIKey,
})
resp, err := http.Post(c.config.ServerURL+"/rotate", "application/json", bytes.NewBuffer(reqBody))
if err != nil {
return fmt.Errorf("failed to request rotation: %v", err)
}
defer resp.Body.Close()
var rotateResp struct {
Success bool `json:"success"`
DeletedCount int `json:"deleted_count"`
ReclaimedBytes int64 `json:"reclaimed_bytes"`
}
if err := json.NewDecoder(resp.Body).Decode(&rotateResp); err != nil {
return fmt.Errorf("failed to decode response: %v", err)
}
if !rotateResp.Success {
return fmt.Errorf("rotation failed")
}
fmt.Printf("✓ Rotation complete\n")
fmt.Printf(" Deleted: %d snapshots\n", rotateResp.DeletedCount)
fmt.Printf(" Freed: %.2f GB\n", float64(rotateResp.ReclaimedBytes)/(1024*1024*1024))
return nil
}
// ServerRotationPolicy represents the rotation policy response from the server
type ServerRotationPolicy struct {
Success bool `json:"success"`
Message string `json:"message"`
RotationPolicy *SnapshotPolicy `json:"rotation_policy"`
ServerManaged bool `json:"server_managed"`
}
// GetRotationPolicy fetches the rotation policy from the server.
// If the server has a policy configured for this client, it must be used.
// Returns the policy and whether it's server-managed (mandatory).
func (c *Client) GetRotationPolicy() (*ServerRotationPolicy, error) {
url := fmt.Sprintf("%s/rotation-policy?client_id=%s&api_key=%s",
c.config.ServerURL, c.config.ClientID, c.config.APIKey)
resp, err := http.Get(url)
if err != nil {
return nil, fmt.Errorf("failed to get rotation policy: %v", err)
}
defer resp.Body.Close()
var policyResp ServerRotationPolicy
if err := json.NewDecoder(resp.Body).Decode(&policyResp); err != nil {
return nil, fmt.Errorf("failed to decode response: %v", err)
}
if !policyResp.Success {
return nil, fmt.Errorf("failed to get rotation policy: %s", policyResp.Message)
}
return &policyResp, nil
}
// ChangePassword changes the client's API key on the server.
// Requires the current API key for authentication and the new key.
func (c *Client) ChangePassword(newAPIKey string) error {
reqBody, _ := json.Marshal(map[string]string{
"client_id": c.config.ClientID,
"current_key": c.config.APIKey,
"new_key": newAPIKey,
})
resp, err := http.Post(c.config.ServerURL+"/client/change-password", "application/json", bytes.NewBuffer(reqBody))
if err != nil {
return fmt.Errorf("failed to change password: %v", err)
}
defer resp.Body.Close()
var result struct {
Success bool `json:"success"`
Message string `json:"message"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return fmt.Errorf("failed to decode response: %v", err)
}
if !result.Success {
return fmt.Errorf("failed to change password: %s", result.Message)
}
// Update local config with new key
c.config.APIKey = newAPIKey
fmt.Printf("✓ Password changed successfully\n")
return nil
}