diff --git a/cmd/zfs-client/main.go b/cmd/zfs-client/main.go index 0f5a4b0..3ebace7 100644 --- a/cmd/zfs-client/main.go +++ b/cmd/zfs-client/main.go @@ -1,5 +1,4 @@ -// Command zfs-client is the CLI tool for creating and uploading ZFS snapshots. -// It provides commands for backup, status checking, snapshot rotation, and incremental backups. +// Command zfs-client is a simple CLI tool for creating and sending ZFS snapshots. package main import ( @@ -22,190 +21,29 @@ func main() { command := os.Args[1] switch command { - case "backup": - // Default: create manual backup (full or incremental) - fmt.Println("=== Creating and sending backup ===\n") + case "snap", "snapshot": + // Create snapshot and send to server (auto full/incremental) + fmt.Println("=== Creating and sending snapshot ===\n") - snapshot, err := c.CreateSnapshot() + snapshot, err := c.CreateAndSend() if err != nil { - fmt.Printf("Error creating snapshot: %v\n", err) + fmt.Printf("Error: %v\n", err) os.Exit(1) } - if err := c.SendSnapshot(snapshot); err != nil { - fmt.Printf("Error sending snapshot: %v\n", err) - os.Exit(1) - } - - fmt.Println("\n✓ Backup completed successfully!") - - case "backup-full": - // Force full backup (no incremental) - fmt.Println("=== Creating full backup ===\n") - - snapshot, err := c.CreateSnapshot() - if err != nil { - fmt.Printf("Error creating snapshot: %v\n", err) - os.Exit(1) - } - - if err := c.SendIncremental(snapshot, ""); err != nil { - fmt.Printf("Error sending snapshot: %v\n", err) - os.Exit(1) - } - - // Create bookmark for future incremental backups - if err := c.CreateBookmark(snapshot); err != nil { - fmt.Printf("Warning: failed to create bookmark: %v\n", err) - } - - fmt.Println("\n✓ Full backup completed successfully!") - - case "backup-incremental": - // Incremental backup from last bookmark - fmt.Println("=== Creating incremental backup ===\n") - - // Check for existing bookmark - lastBookmark, err := c.GetLastBookmark() - if err != nil { - fmt.Printf("Error checking bookmarks: %v\n", err) - os.Exit(1) - } - - if lastBookmark == "" { - fmt.Println("No existing bookmark found. Use 'backup-full' for initial backup.") - os.Exit(1) - } - - snapshot, err := c.CreateSnapshot() - if err != nil { - fmt.Printf("Error creating snapshot: %v\n", err) - os.Exit(1) - } - - if err := c.SendIncremental(snapshot, lastBookmark); err != nil { - fmt.Printf("Error sending incremental snapshot: %v\n", err) - os.Exit(1) - } - - // Create bookmark for future incremental backups - if err := c.CreateBookmark(snapshot); err != nil { - fmt.Printf("Warning: failed to create bookmark: %v\n", err) - } - - fmt.Println("\n✓ Incremental backup completed successfully!") - - case "snapshot": - // Create typed snapshots (hourly, daily, weekly, monthly) - if len(os.Args) < 3 { - fmt.Println("Usage: zfs-client snapshot ") - os.Exit(1) - } - - snapType := client.SnapshotType(os.Args[2]) - switch snapType { - case client.SnapshotHourly, client.SnapshotDaily, client.SnapshotWeekly, client.SnapshotMonthly: - // Valid type - default: - fmt.Printf("Invalid snapshot type: %s\n", snapType) - fmt.Println("Valid types: hourly, daily, weekly, monthly") - os.Exit(1) - } - - fmt.Printf("=== Creating %s snapshot ===\n\n", snapType) - - snapshot, err := c.CreateSnapshotWithType(snapType) - if err != nil { - fmt.Printf("Error creating snapshot: %v\n", err) - os.Exit(1) - } - - // Check for existing bookmark for incremental - lastBookmark, _ := c.GetLastBookmark() - - if err := c.SendIncremental(snapshot, lastBookmark); err != nil { - fmt.Printf("Error sending snapshot: %v\n", err) - os.Exit(1) - } - - // Create bookmark - if err := c.CreateBookmark(snapshot); err != nil { - fmt.Printf("Warning: failed to create bookmark: %v\n", err) - } - - // Rotate local snapshots using server policy if available - policy, err := getRotationPolicy(c) - if err != nil { - fmt.Printf("Warning: failed to get rotation policy: %v\n", err) - policy = client.DefaultPolicy() - } - if err := c.RotateLocalSnapshots(policy); err != nil { - fmt.Printf("Warning: failed to rotate snapshots: %v\n", err) - } - - fmt.Printf("\n✓ %s snapshot completed successfully!\n", snapType) - - case "rotate": - // Rotate local snapshots using server policy if available - fmt.Println("=== Rotating local snapshots ===\n") - - policy, err := getRotationPolicy(c) - if err != nil { - fmt.Printf("Warning: failed to get rotation policy: %v\n", err) - policy = client.DefaultPolicy() - } - if err := c.RotateLocalSnapshots(policy); err != nil { - fmt.Printf("Error rotating snapshots: %v\n", err) - os.Exit(1) - } - - fmt.Println("\n✓ Rotation completed!") - - case "rotate-remote": - // Request server to rotate remote snapshots - if err := c.RequestRotation(); err != nil { - fmt.Printf("Error requesting rotation: %v\n", err) - os.Exit(1) + if snapshot.FullBackup { + fmt.Println("\n✓ Full backup completed!") + } else { + fmt.Println("\n✓ Incremental backup completed!") } case "status": + // Check server connection and quota if err := c.GetStatus(); err != nil { - fmt.Printf("Error getting status: %v\n", err) - os.Exit(1) - } - - case "bookmarks": - // List bookmarks - fmt.Println("=== ZFS Bookmarks ===\n") - - bookmark, err := c.GetLastBookmark() - if err != nil { fmt.Printf("Error: %v\n", err) os.Exit(1) } - if bookmark == "" { - fmt.Println("No bookmarks found") - } else { - fmt.Printf("Last bookmark: %s\n", bookmark) - } - - case "change-password": - // Change client API key/password - if len(os.Args) < 3 { - fmt.Println("Usage: zfs-client change-password ") - os.Exit(1) - } - newKey := os.Args[2] - - fmt.Println("=== Changing API Key ===\n") - if err := c.ChangePassword(newKey); err != nil { - fmt.Printf("Error: %v\n", err) - os.Exit(1) - } - fmt.Println("\n✓ API key changed successfully!") - fmt.Println("Update your .env file with the new API_KEY value.") - case "help", "-h", "--help": printUsage() @@ -216,56 +54,26 @@ func main() { } } -// getRotationPolicy fetches the rotation policy from the server. -// If the server has a policy configured, it must be used. -// Otherwise, the default policy is returned. -func getRotationPolicy(c *client.Client) (*client.SnapshotPolicy, error) { - serverPolicy, err := c.GetRotationPolicy() - if err != nil { - return nil, err - } - - if serverPolicy.ServerManaged && serverPolicy.RotationPolicy != nil { - fmt.Println(" Using server-managed rotation policy") - return serverPolicy.RotationPolicy, nil - } - - // No server policy, use default - fmt.Println(" Using default rotation policy") - return client.DefaultPolicy(), nil -} - func printUsage() { - fmt.Println("ZFS Snapshot Backup Client") + fmt.Println("ZFS Snapshot Backup Client - Simple Version") fmt.Println("\nUsage: zfs-client [command]") fmt.Println("\nCommands:") - fmt.Println(" backup - Create snapshot and send (auto incremental if bookmark exists)") - fmt.Println(" backup-full - Create full backup (no incremental)") - fmt.Println(" backup-incremental - Create incremental backup from last bookmark") - fmt.Println(" snapshot - Create typed snapshot (hourly|daily|weekly|monthly)") - fmt.Println(" rotate - Rotate local snapshots based on retention policy") - fmt.Println(" rotate-remote - Request server to rotate old remote snapshots") - fmt.Println(" status - Check server status and quota") - fmt.Println(" bookmarks - List ZFS bookmarks") - fmt.Println(" change-password - Change client API key") - fmt.Println(" help - Show this help message") - fmt.Println("\nSnapshot Retention Policy (default):") - fmt.Println(" Hourly: 24 snapshots") - fmt.Println(" Daily: 7 snapshots") - fmt.Println(" Weekly: 4 snapshots") - fmt.Println(" Monthly: 12 snapshots") + fmt.Println(" snap - Create snapshot and send to server (auto full/incremental)") + fmt.Println(" status - Check server connection and quota") + fmt.Println(" help - Show this help message") fmt.Println("\nEnvironment Variables (can be set in .env file):") fmt.Println(" CLIENT_ID - Client identifier (default: client1)") fmt.Println(" API_KEY - API key for authentication (default: secret123)") fmt.Println(" SERVER_URL - Backup server URL (default: http://localhost:8080)") fmt.Println(" LOCAL_DATASET - ZFS dataset to backup (default: tank/data)") fmt.Println(" COMPRESS - Enable LZ4 compression (default: true)") - fmt.Println(" STORAGE_TYPE - Storage type: s3 or local (default: s3)") + fmt.Println("\nS3 Configuration (for direct S3 uploads):") + fmt.Println(" S3_ENDPOINT - S3 endpoint URL (e.g., https://s3.amazonaws.com)") + fmt.Println(" S3_REGION - AWS region (default: us-east-1)") + fmt.Println(" S3_BUCKET - S3 bucket name (default: zfs-backups)") + fmt.Println(" S3_ACCESS_KEY - AWS access key") + fmt.Println(" S3_SECRET_KEY - AWS secret key") fmt.Println("\nExamples:") - fmt.Println(" zfs-client backup") - fmt.Println(" zfs-client backup-full") - fmt.Println(" zfs-client snapshot hourly") - fmt.Println(" zfs-client rotate") - fmt.Println(" zfs-client change-password mynewsecretkey") - fmt.Println(" CLIENT_ID=myclient zfs-client backup") + fmt.Println(" zfs-client snap") + fmt.Println(" zfs-client status") } diff --git a/go.mod b/go.mod index 9a7f327..3bf37f6 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,25 @@ require ( require ( github.com/a-h/templ v0.3.977 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.7 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.7 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect + github.com/aws/smithy-go v1.24.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-ini/ini v1.67.0 // indirect diff --git a/go.sum b/go.sum index 454fcdc..2d8d742 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,43 @@ github.com/a-h/templ v0.3.977 h1:kiKAPXTZE2Iaf8JbtM21r54A8bCNsncrfnokZZSrSDg= github.com/a-h/templ v0.3.977/go.mod h1:oCZcnKRf5jjsGpf2yELzQfodLphd2mwecwG4Crk5HBo= +github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU= +github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4= +github.com/aws/aws-sdk-go-v2/config v1.32.7 h1:vxUyWGUwmkQ2g19n7JY/9YL8MfAIl7bTesIUykECXmY= +github.com/aws/aws-sdk-go-v2/config v1.32.7/go.mod h1:2/Qm5vKUU/r7Y+zUk/Ptt2MDAEKAfUtKc1+3U1Mo3oY= +github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUTvZKeLx4tH6PGQc8= +github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 h1:gd84Omyu9JLriJVCbGApcLzVR3XtmC4ZDPcAI6Ftvds= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13/go.mod h1:sTGThjphYE4Ohw8vJiRStAcu3rbjtXRsdNB0TvZ5wwo= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 h1:5fFjR/ToSOzB2OQ/XqWpZBmNvmP/pJ1jOWYlFDJTjRQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.6/go.mod h1:qgFDZQSD/Kys7nJnVqYlWKnh0SSdMjAi0uSwON4wgYQ= +github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= +github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= diff --git a/internal/client/client.go b/internal/client/client.go index fa4d328..ab1b697 100644 --- a/internal/client/client.go +++ b/internal/client/client.go @@ -4,6 +4,7 @@ package client import ( "bytes" + "context" "encoding/json" "fmt" "io" @@ -13,6 +14,10 @@ import ( "strings" "time" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/mistifyio/go-zfs" "github.com/pierrec/lz4/v4" ) @@ -109,10 +114,31 @@ func (c *Client) SendSnapshot(snapshot *zfs.Dataset) error { return c.sendViaZFS(snapshot, uploadResp.StorageKey) } -// streamToS3 streams a ZFS snapshot to S3 storage via HTTP. +// streamToS3 streams a ZFS snapshot to S3 storage using AWS SDK. // The snapshot is optionally compressed with LZ4 before transmission. func (c *Client) streamToS3(snapshot *zfs.Dataset, uploadURL, storageKey string) error { - fmt.Printf("→ Streaming snapshot to S3...\n") + fmt.Printf("→ Uploading snapshot to S3...\n") + + // Create AWS config + awsCfg, err := config.LoadDefaultConfig(context.TODO(), + config.WithRegion(c.config.S3Region), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + c.config.S3AccessKey, + c.config.S3SecretKey, + "", + )), + ) + if err != nil { + return fmt.Errorf("failed to load AWS config: %v", err) + } + + // Create S3 client + s3Client := s3.NewFromConfig(awsCfg, func(o *s3.Options) { + if c.config.S3Endpoint != "" { + o.BaseEndpoint = aws.String(c.config.S3Endpoint) + o.UsePathStyle = true // Required for MinIO compatible storage + } + }) // Create ZFS send command cmd := exec.Command("zfs", "send", snapshot.Name) @@ -145,57 +171,24 @@ func (c *Client) streamToS3(snapshot *zfs.Dataset, uploadURL, storageKey string) reader = pr } - // Create HTTP request - req, err := http.NewRequest("POST", c.config.ServerURL+uploadURL, reader) - if err != nil { - return fmt.Errorf("failed to create request: %v", err) - } - - // Set required headers - req.Header.Set("X-API-Key", c.config.APIKey) - req.Header.Set("X-Storage-Key", storageKey) - req.Header.Set("X-Dataset-Name", c.config.LocalDataset) - req.Header.Set("X-Compressed", fmt.Sprintf("%v", c.config.Compress)) - req.Header.Set("Content-Type", "application/octet-stream") - - // Send request with no timeout for large uploads - client := &http.Client{ - Timeout: 0, - } - - httpResp, err := client.Do(req) - if err != nil { - cmd.Process.Kill() - return fmt.Errorf("failed to upload: %v", err) - } - defer httpResp.Body.Close() - - if httpResp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(httpResp.Body) - return fmt.Errorf("upload failed with status %d: %s", httpResp.StatusCode, body) - } + // Upload to S3 using PutObject + _, err = s3Client.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(c.config.S3Bucket), + Key: aws.String(storageKey), + Body: reader, + ContentType: aws.String("application/octet-stream"), + }) + // Wait for zfs send to complete if err := cmd.Wait(); err != nil { return fmt.Errorf("zfs send failed: %v", err) } - // Parse response - var result struct { - Success bool `json:"success"` - Message string `json:"message"` - Size int64 `json:"size"` + if err != nil { + return fmt.Errorf("failed to upload to S3: %v", err) } - if err := json.NewDecoder(httpResp.Body).Decode(&result); err != nil { - return fmt.Errorf("failed to decode response: %v", err) - } - - if !result.Success { - return fmt.Errorf("upload failed: %s", result.Message) - } - - fmt.Printf("✓ Snapshot uploaded successfully!\n") - fmt.Printf(" Size: %.2f MB\n", float64(result.Size)/(1024*1024)) + fmt.Printf("✓ Snapshot uploaded to S3 successfully!\n") return nil } @@ -234,6 +227,55 @@ func (c *Client) sendViaZFS(snapshot *zfs.Dataset, receivePath string) error { return nil } +// SnapshotResult contains the result of a snapshot creation and send operation. +type SnapshotResult struct { + FullBackup bool + Snapshot *zfs.Dataset +} + +// CreateAndSend creates a snapshot and sends it to the backup server. +// It automatically detects if this is a full or incremental backup: +// - If no bookmark exists, does a full backup +// - If bookmark exists, does an incremental backup from the bookmark +func (c *Client) CreateAndSend() (*SnapshotResult, error) { + // Check for existing bookmark to determine backup type + lastBookmark, err := c.GetLastBookmark() + if err != nil { + return nil, fmt.Errorf("failed to check bookmarks: %v", err) + } + + // Create new snapshot + snapshot, err := c.CreateSnapshot() + if err != nil { + return nil, fmt.Errorf("failed to create snapshot: %v", err) + } + + isFullBackup := lastBookmark == "" + if isFullBackup { + fmt.Println("→ No previous backup found, doing FULL backup...") + // Send as full (no base) + if err := c.SendIncremental(snapshot, ""); err != nil { + return nil, fmt.Errorf("failed to send snapshot: %v", err) + } + } else { + fmt.Printf("→ Found previous backup, doing INCREMENTAL from %s...", lastBookmark) + // Send as incremental from bookmark + if err := c.SendIncremental(snapshot, lastBookmark); err != nil { + return nil, fmt.Errorf("failed to send incremental: %v", err) + } + } + + // Create bookmark for future incremental backups + if err := c.CreateBookmark(snapshot); err != nil { + fmt.Printf("Warning: failed to create bookmark: %v\n", err) + } + + return &SnapshotResult{ + FullBackup: isFullBackup, + Snapshot: snapshot, + }, nil +} + // GetStatus retrieves and displays the client's backup status from the server. // Shows storage usage, quota, and snapshot count. func (c *Client) GetStatus() error { @@ -273,106 +315,3 @@ func (c *Client) GetStatus() error { return nil } - -// RequestRotation asks the server to rotate old snapshots. -// This deletes the oldest snapshots to free up space. -func (c *Client) RequestRotation() error { - reqBody, _ := json.Marshal(map[string]string{ - "client_id": c.config.ClientID, - "api_key": c.config.APIKey, - }) - - resp, err := http.Post(c.config.ServerURL+"/rotate", "application/json", bytes.NewBuffer(reqBody)) - if err != nil { - return fmt.Errorf("failed to request rotation: %v", err) - } - defer resp.Body.Close() - - var rotateResp struct { - Success bool `json:"success"` - DeletedCount int `json:"deleted_count"` - ReclaimedBytes int64 `json:"reclaimed_bytes"` - } - - if err := json.NewDecoder(resp.Body).Decode(&rotateResp); err != nil { - return fmt.Errorf("failed to decode response: %v", err) - } - - if !rotateResp.Success { - return fmt.Errorf("rotation failed") - } - - fmt.Printf("✓ Rotation complete\n") - fmt.Printf(" Deleted: %d snapshots\n", rotateResp.DeletedCount) - fmt.Printf(" Freed: %.2f GB\n", float64(rotateResp.ReclaimedBytes)/(1024*1024*1024)) - - return nil -} - -// ServerRotationPolicy represents the rotation policy response from the server -type ServerRotationPolicy struct { - Success bool `json:"success"` - Message string `json:"message"` - RotationPolicy *SnapshotPolicy `json:"rotation_policy"` - ServerManaged bool `json:"server_managed"` -} - -// GetRotationPolicy fetches the rotation policy from the server. -// If the server has a policy configured for this client, it must be used. -// Returns the policy and whether it's server-managed (mandatory). -func (c *Client) GetRotationPolicy() (*ServerRotationPolicy, error) { - url := fmt.Sprintf("%s/rotation-policy?client_id=%s&api_key=%s", - c.config.ServerURL, c.config.ClientID, c.config.APIKey) - - resp, err := http.Get(url) - if err != nil { - return nil, fmt.Errorf("failed to get rotation policy: %v", err) - } - defer resp.Body.Close() - - var policyResp ServerRotationPolicy - if err := json.NewDecoder(resp.Body).Decode(&policyResp); err != nil { - return nil, fmt.Errorf("failed to decode response: %v", err) - } - - if !policyResp.Success { - return nil, fmt.Errorf("failed to get rotation policy: %s", policyResp.Message) - } - - return &policyResp, nil -} - -// ChangePassword changes the client's API key on the server. -// Requires the current API key for authentication and the new key. -func (c *Client) ChangePassword(newAPIKey string) error { - reqBody, _ := json.Marshal(map[string]string{ - "client_id": c.config.ClientID, - "current_key": c.config.APIKey, - "new_key": newAPIKey, - }) - - resp, err := http.Post(c.config.ServerURL+"/client/change-password", "application/json", bytes.NewBuffer(reqBody)) - if err != nil { - return fmt.Errorf("failed to change password: %v", err) - } - defer resp.Body.Close() - - var result struct { - Success bool `json:"success"` - Message string `json:"message"` - } - - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return fmt.Errorf("failed to decode response: %v", err) - } - - if !result.Success { - return fmt.Errorf("failed to change password: %s", result.Message) - } - - // Update local config with new key - c.config.APIKey = newAPIKey - - fmt.Printf("✓ Password changed successfully\n") - return nil -} diff --git a/internal/client/config.go b/internal/client/config.go index 81c3823..546e9b2 100644 --- a/internal/client/config.go +++ b/internal/client/config.go @@ -21,6 +21,16 @@ type Config struct { LocalDataset string `json:"local_dataset"` // Compress enables LZ4 compression for transfers Compress bool `json:"compress"` + // S3Endpoint is the S3 endpoint URL (optional, for direct S3 uploads) + S3Endpoint string `json:"s3_endpoint"` + // S3Region is the AWS region + S3Region string `json:"s3_region"` + // S3Bucket is the S3 bucket name + S3Bucket string `json:"s3_bucket"` + // S3AccessKey is the AWS access key + S3AccessKey string `json:"s3_access_key"` + // S3SecretKey is the AWS secret key + S3SecretKey string `json:"s3_secret_key"` } // LoadConfig loads client configuration from environment variables and .env file. @@ -35,6 +45,11 @@ func LoadConfig() *Config { ServerURL: getEnv("SERVER_URL", "http://backup-server:8080"), LocalDataset: getEnv("LOCAL_DATASET", "tank/data"), Compress: getEnv("COMPRESS", "true") == "true", + S3Endpoint: getEnv("S3_ENDPOINT", ""), + S3Region: getEnv("S3_REGION", "us-east-1"), + S3Bucket: getEnv("S3_BUCKET", "zfs-backups"), + S3AccessKey: getEnv("S3_ACCESS_KEY", ""), + S3SecretKey: getEnv("S3_SECRET_KEY", ""), } } diff --git a/internal/client/snapshot.go b/internal/client/snapshot.go index c52d087..cb1095f 100644 --- a/internal/client/snapshot.go +++ b/internal/client/snapshot.go @@ -1,77 +1,27 @@ // Package client provides ZFS snapshot backup client functionality. -// This file contains snapshot management functions including creation, -// bookmarking, and rotation similar to zfs-auto-snapshot. +// This file contains snapshot management functions for creating and sending snapshots. package client import ( "bytes" + "context" "encoding/json" "fmt" "io" "net/http" "os" "os/exec" - "sort" "strings" "time" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/mistifyio/go-zfs" "github.com/pierrec/lz4/v4" ) -// SnapshotPolicy defines retention settings for automatic snapshots. -type SnapshotPolicy struct { - // KeepHourly is the number of hourly snapshots to keep - KeepHourly int - // KeepDaily is the number of daily snapshots to keep - KeepDaily int - // KeepWeekly is the number of weekly snapshots to keep - KeepWeekly int - // KeepMonthly is the number of monthly snapshots to keep - KeepMonthly int -} - -// DefaultPolicy returns the default snapshot retention policy. -func DefaultPolicy() *SnapshotPolicy { - return &SnapshotPolicy{ - KeepHourly: 24, - KeepDaily: 7, - KeepWeekly: 4, - KeepMonthly: 12, - } -} - -// SnapshotType represents the type of snapshot (hourly, daily, etc.) -type SnapshotType string - -const ( - SnapshotHourly SnapshotType = "hourly" - SnapshotDaily SnapshotType = "daily" - SnapshotWeekly SnapshotType = "weekly" - SnapshotMonthly SnapshotType = "monthly" - SnapshotManual SnapshotType = "manual" -) - -// CreateSnapshotWithType creates a snapshot with a specific type label. -// The snapshot name follows the pattern: zfs-backup-- -func (c *Client) CreateSnapshotWithType(snapshotType SnapshotType) (*zfs.Dataset, error) { - ds, err := zfs.GetDataset(c.config.LocalDataset) - if err != nil { - return nil, fmt.Errorf("failed to get dataset: %v", err) - } - - timestamp := time.Now().Format("2006-01-02_15-04-05") - snapshotName := fmt.Sprintf("zfs-backup-%s-%s", snapshotType, timestamp) - - snapshot, err := ds.Snapshot(snapshotName, false) - if err != nil { - return nil, fmt.Errorf("failed to create snapshot: %v", err) - } - - fmt.Printf("✓ Created %s snapshot: %s@%s\n", snapshotType, c.config.LocalDataset, snapshotName) - return snapshot, nil -} - // CreateBookmark creates a ZFS bookmark from a snapshot. // Bookmarks allow incremental sends even after the source snapshot is deleted. func (c *Client) CreateBookmark(snapshot *zfs.Dataset) error { @@ -194,9 +144,30 @@ func (c *Client) SendIncremental(snapshot *zfs.Dataset, base string) error { return c.sendIncrementalViaZFS(snapshot, base, uploadResp.StorageKey) } -// streamIncrementalToS3 streams an incremental ZFS snapshot to S3. +// streamIncrementalToS3 streams an incremental ZFS snapshot to S3 using AWS SDK. func (c *Client) streamIncrementalToS3(snapshot *zfs.Dataset, base, uploadURL, storageKey string) error { - fmt.Printf("→ Streaming snapshot to S3...\n") + fmt.Printf("→ Uploading snapshot to S3...\n") + + // Create AWS config + awsCfg, err := config.LoadDefaultConfig(context.TODO(), + config.WithRegion(c.config.S3Region), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + c.config.S3AccessKey, + c.config.S3SecretKey, + "", + )), + ) + if err != nil { + return fmt.Errorf("failed to load AWS config: %v", err) + } + + // Create S3 client + s3Client := s3.NewFromConfig(awsCfg, func(o *s3.Options) { + if c.config.S3Endpoint != "" { + o.BaseEndpoint = aws.String(c.config.S3Endpoint) + o.UsePathStyle = true // Required for MinIO compatible storage + } + }) // Create ZFS send command var cmd *exec.Cmd @@ -238,61 +209,24 @@ func (c *Client) streamIncrementalToS3(snapshot *zfs.Dataset, base, uploadURL, s reader = pr } - // Create HTTP request - req, err := http.NewRequest("POST", c.config.ServerURL+uploadURL, reader) - if err != nil { - return fmt.Errorf("failed to create request: %v", err) - } - - // Set required headers - req.Header.Set("X-API-Key", c.config.APIKey) - req.Header.Set("X-Storage-Key", storageKey) - req.Header.Set("X-Dataset-Name", c.config.LocalDataset) - req.Header.Set("X-Compressed", fmt.Sprintf("%v", c.config.Compress)) - req.Header.Set("X-Incremental", fmt.Sprintf("%v", base != "")) - if base != "" { - req.Header.Set("X-Base-Snapshot", base) - } - req.Header.Set("Content-Type", "application/octet-stream") - - // Send request with no timeout for large uploads - client := &http.Client{ - Timeout: 0, - } - - httpResp, err := client.Do(req) - if err != nil { - cmd.Process.Kill() - return fmt.Errorf("failed to upload: %v", err) - } - defer httpResp.Body.Close() - - if httpResp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(httpResp.Body) - return fmt.Errorf("upload failed with status %d: %s", httpResp.StatusCode, body) - } + // Upload to S3 using PutObject + _, err = s3Client.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(c.config.S3Bucket), + Key: aws.String(storageKey), + Body: reader, + ContentType: aws.String("application/octet-stream"), + }) + // Wait for zfs send to complete if err := cmd.Wait(); err != nil { return fmt.Errorf("zfs send failed: %v", err) } - // Parse response - var result struct { - Success bool `json:"success"` - Message string `json:"message"` - Size int64 `json:"size"` + if err != nil { + return fmt.Errorf("failed to upload to S3: %v", err) } - if err := json.NewDecoder(httpResp.Body).Decode(&result); err != nil { - return fmt.Errorf("failed to decode response: %v", err) - } - - if !result.Success { - return fmt.Errorf("upload failed: %s", result.Message) - } - - fmt.Printf("✓ Snapshot uploaded successfully!\n") - fmt.Printf(" Size: %.2f MB\n", float64(result.Size)/(1024*1024)) + fmt.Printf("✓ Snapshot uploaded to S3 successfully!\n") return nil } @@ -300,7 +234,7 @@ func (c *Client) streamIncrementalToS3(snapshot *zfs.Dataset, base, uploadURL, s // sendIncrementalViaZFS sends an incremental snapshot via ZFS send/receive over SSH. // This method is used when the server uses local ZFS storage. func (c *Client) sendIncrementalViaZFS(snapshot *zfs.Dataset, base, receivePath string) error { - fmt.Printf("-> Sending via ZFS send/receive...\n") + fmt.Printf("→ Sending via ZFS send/receive...\n") // Extract server host from URL serverHost := c.config.ServerURL @@ -337,93 +271,6 @@ func (c *Client) sendIncrementalViaZFS(snapshot *zfs.Dataset, base, receivePath return fmt.Errorf("failed to send snapshot: %v", err) } - fmt.Printf("Snapshot sent successfully!\n") + fmt.Printf("✓ Snapshot sent successfully!\n") return nil } - -// RotateLocalSnapshots removes old snapshots based on the retention policy. -// This is similar to zfs-auto-snapshot's rotation behavior. -func (c *Client) RotateLocalSnapshots(policy *SnapshotPolicy) error { - ds, err := zfs.GetDataset(c.config.LocalDataset) - if err != nil { - return fmt.Errorf("failed to get dataset: %v", err) - } - - snapshots, err := ds.Snapshots() - if err != nil { - return fmt.Errorf("failed to list snapshots: %v", err) - } - - // Group snapshots by type - groups := make(map[SnapshotType][]*zfs.Dataset) - for _, snap := range snapshots { - snapType := parseSnapshotType(snap.Name) - groups[snapType] = append(groups[snapType], snap) - } - - // Apply retention policy - deletedCount := 0 - keepCount := map[SnapshotType]int{ - SnapshotHourly: policy.KeepHourly, - SnapshotDaily: policy.KeepDaily, - SnapshotWeekly: policy.KeepWeekly, - SnapshotMonthly: policy.KeepMonthly, - SnapshotManual: -1, // Keep all manual snapshots - } - - for snapType, snaps := range groups { - maxKeep := keepCount[snapType] - if maxKeep < 0 { - continue // Keep all - } - - // Sort by creation time (oldest first) - sortSnapshotsByTime(snaps) - - // Delete oldest snapshots exceeding the limit - if len(snaps) > maxKeep { - toDelete := snaps[:len(snaps)-maxKeep] - for _, snap := range toDelete { - fmt.Printf(" Deleting old snapshot: %s\n", snap.Name) - if err := snap.Destroy(zfs.DestroyDefault); err != nil { - fmt.Printf(" Warning: failed to delete %s: %v\n", snap.Name, err) - } else { - deletedCount++ - } - } - } - } - - if deletedCount > 0 { - fmt.Printf("✓ Rotated %d local snapshots\n", deletedCount) - } - - return nil -} - -// parseSnapshotType extracts the snapshot type from the snapshot name. -func parseSnapshotType(name string) SnapshotType { - if strings.Contains(name, "hourly") { - return SnapshotHourly - } - if strings.Contains(name, "daily") { - return SnapshotDaily - } - if strings.Contains(name, "weekly") { - return SnapshotWeekly - } - if strings.Contains(name, "monthly") { - return SnapshotMonthly - } - return SnapshotManual -} - -// sortSnapshotsByTime sorts snapshots by creation time (oldest first). -// Uses the snapshot name which contains timestamp for sorting. -func sortSnapshotsByTime(snaps []*zfs.Dataset) { - sort.Slice(snaps, func(i, j int) bool { - // Extract timestamp from snapshot name for comparison - // Names are like: dataset@zfs-backup-hourly-2006-01-02_15-04-05 - return snaps[i].Name < snaps[j].Name - }) -} diff --git a/internal/server/templates/pages/admin.templ b/internal/server/templates/pages/admin.templ index d47de89..a8fdde2 100644 --- a/internal/server/templates/pages/admin.templ +++ b/internal/server/templates/pages/admin.templ @@ -260,4 +260,5 @@ templ ClientPasswordModal() { // AdminScripts renders the JavaScript for the admin panel templ AdminScripts() { + } diff --git a/internal/server/templates/static/admin.js b/internal/server/templates/static/admin.js index afbc870..2cd2a49 100644 --- a/internal/server/templates/static/admin.js +++ b/internal/server/templates/static/admin.js @@ -19,6 +19,30 @@ async function logout() { location.reload(); } +// Toggle dark/light theme +function toggleTheme() { + const html = document.documentElement; + const isDark = html.classList.contains('dark'); + + if (isDark) { + html.classList.remove('dark'); + localStorage.setItem('theme', 'light'); + } else { + html.classList.add('dark'); + localStorage.setItem('theme', 'dark'); + } +} + +// Initialize theme on load +function initTheme() { + const savedTheme = localStorage.getItem('theme'); + const prefersDark = window.matchMedia('(prefers-color-scheme: dark)').matches; + + if (savedTheme === 'dark' || (!savedTheme && prefersDark)) { + document.documentElement.classList.add('dark'); + } +} + // Load stats async function loadStats() { try { diff --git a/readme.md b/readme.md index 5c59e3c..cc6b926 100644 --- a/readme.md +++ b/readme.md @@ -4,13 +4,14 @@ A distributed ZFS snapshot management system with S3-compatible storage support. ## Features -- **S3 Storage Support**: Store snapshots in any S3-compatible storage (AWS S3, MinIO, Backblaze B2, Wasabi, DigitalOcean Spaces) +- **S3 Storage Support**: Store snapshots in any S3-compatible storage using AWS SDK v2 (AWS S3, MinIO, Backblaze B2, Wasabi, DigitalOcean Spaces) - **Local ZFS Storage**: Option to use local ZFS datasets for maximum performance - **Multi-client Architecture**: Support for multiple clients with isolated storage and per-client quotas -- **Automatic Compression**: Gzip compression for reduced storage costs +- **Automatic Compression**: LZ4 compression for reduced storage costs and faster transfers - **Snapshot Rotation**: Automatic cleanup of old snapshots based on quota - **Server-Managed Rotation Policies**: Centralized control of client rotation policies - clients must use server-configured retention settings - **API Key Authentication**: Secure client-server communication +- **Simple CLI**: Just use `zfs-client snap` to backup - automatically handles full/incremental ## Project Structure @@ -93,11 +94,19 @@ API_KEY=secret123 SERVER_URL=http://backup-server:8080 LOCAL_DATASET=tank/data COMPRESS=true + +# Optional: Direct S3 upload (bypasses server storage) +S3_ENDPOINT=https://s3.amazonaws.com +S3_REGION=us-east-1 +S3_BUCKET=zfs-backups +S3_ACCESS_KEY=your_access_key +S3_SECRET_KEY=your_secret_key ``` > **Important**: > - The `API_KEY` in the client `.env` file must be the **raw (unhashed)** key. The server stores the SHA-256 hash in the database. > - **Storage type is determined by the server**, not the client. The server decides whether to use S3 or local ZFS storage based on its configuration. +> - The client automatically handles full vs incremental backups based on whether a bookmark exists. ### Restore Tool Configuration @@ -128,52 +137,18 @@ zfs-server ### Client Commands -The `zfs-client` tool provides the following commands for managing ZFS snapshots: +The `zfs-client` tool provides simple commands for creating and sending ZFS snapshots: -#### `backup` -Creates a snapshot and sends it to the server. Automatically uses incremental backup if a bookmark exists. +#### `snap` +Creates a snapshot and sends it to the server. Automatically detects if this is the first backup (full) or subsequent backup (incremental). ```bash -zfs-client backup +zfs-client snap ``` -#### `backup-full` -Forces a full backup (no incremental). Use for the initial backup or when you want to resend the complete dataset. +On first run, it will print: `→ No previous backup found, doing FULL backup...` -```bash -zfs-client backup-full -``` - -#### `backup-incremental` -Creates an incremental backup from the last bookmark. Requires an existing bookmark from a previous full backup. - -```bash -zfs-client backup-incremental -``` - -#### `snapshot ` -Creates a typed snapshot (hourly, daily, weekly, monthly) with automatic rotation. The rotation policy is fetched from the server if configured. - -```bash -zfs-client snapshot hourly -zfs-client snapshot daily -zfs-client snapshot weekly -zfs-client snapshot monthly -``` - -#### `rotate` -Rotates local snapshots based on the retention policy. If the server has a rotation policy configured, it will be used; otherwise, default values apply. - -```bash -zfs-client rotate -``` - -#### `rotate-remote` -Requests the server to rotate (delete old) remote snapshots to free up storage quota. - -```bash -zfs-client rotate-remote -``` +On subsequent runs, it automatically does incremental backups from the last bookmark. #### `status` Displays the current backup status including storage usage, quota, and snapshot count from the server. @@ -182,13 +157,6 @@ Displays the current backup status including storage usage, quota, and snapshot zfs-client status ``` -#### `bookmarks` -Lists ZFS bookmarks on the local system. Bookmarks are used as reference points for incremental backups. - -```bash -zfs-client bookmarks -``` - #### `help` Shows the help message with all available commands and options. @@ -196,6 +164,23 @@ Shows the help message with all available commands and options. zfs-client help ``` +### Client Configuration + +```env +CLIENT_ID=client1 +API_KEY=secret123 +SERVER_URL=http://backup-server:8080 +LOCAL_DATASET=tank/data +COMPRESS=true + +# Optional: S3 direct upload (bypasses server) +S3_ENDPOINT=https://s3.amazonaws.com +S3_REGION=us-east-1 +S3_BUCKET=zfs-backups +S3_ACCESS_KEY=your_access_key +S3_SECRET_KEY=your_secret_key +``` + ### Restore Tool Commands The `zfs-restore` tool provides commands for listing and restoring snapshots from the backup server: diff --git a/zfs-client b/zfs-client new file mode 100755 index 0000000..dfbffc4 Binary files /dev/null and b/zfs-client differ