goc_marek/zfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add lz4

Browse Source
This commit is contained in:
Marek Goc
2026-02-13 21:50:00 +01:00
parent dbfd99f410
commit 1825b50dec
11 changed files with 276 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
cmd/zfs-client/main.go
View File

@@ -190,6 +190,22 @@ func main() {
fmt.Printf("Last bookmark: %s\n", bookmark)
}
case "change-password":
// Change client API key/password
if len(os.Args) < 3 {
fmt.Println("Usage: zfs-client change-password <new-api-key>")
os.Exit(1)
}
newKey := os.Args[2]
fmt.Println("=== Changing API Key ===\n")
if err := c.ChangePassword(newKey); err != nil {
fmt.Printf("Error: %v\n", err)
os.Exit(1)
}
fmt.Println("\n✓ API key changed successfully!")
fmt.Println("Update your .env file with the new API_KEY value.")
case "help", "-h", "--help":
printUsage()
@@ -231,6 +247,7 @@ func printUsage() {
fmt.Println(" rotate-remote - Request server to rotate old remote snapshots")
fmt.Println(" status - Check server status and quota")
fmt.Println(" bookmarks - List ZFS bookmarks")
fmt.Println(" change-password <new-key> - Change client API key")
fmt.Println(" help - Show this help message")
fmt.Println("\nSnapshot Retention Policy (default):")
fmt.Println(" Hourly: 24 snapshots")
@@ -242,12 +259,13 @@ func printUsage() {
fmt.Println(" API_KEY - API key for authentication (default: secret123)")
fmt.Println(" SERVER_URL - Backup server URL (default: http://localhost:8080)")
fmt.Println(" LOCAL_DATASET - ZFS dataset to backup (default: tank/data)")
fmt.Println(" COMPRESS - Enable gzip compression (default: true)")
fmt.Println(" COMPRESS - Enable LZ4 compression (default: true)")
fmt.Println(" STORAGE_TYPE - Storage type: s3 or local (default: s3)")
fmt.Println("\nExamples:")
fmt.Println(" zfs-client backup")
fmt.Println(" zfs-client backup-full")
fmt.Println(" zfs-client snapshot hourly")
fmt.Println(" zfs-client rotate")
fmt.Println(" zfs-client change-password mynewsecretkey")
fmt.Println(" CLIENT_ID=myclient zfs-client backup")
}

2
cmd/zfs-restore/main.go
View File

@@ -169,7 +169,7 @@ func printUsage() {
fmt.Println(" zfs-restore list")
fmt.Println(" zfs-restore restore 1 tank/restored")
fmt.Println(" zfs-restore latest tank/restored --force")
fmt.Println(" zfs-restore save 2 backup.zfs.gz")
fmt.Println(" zfs-restore save 2 backup.zfs.lz4")
fmt.Println(" zfs-restore mount tank/restored /mnt/restore")
fmt.Println("\nEnvironment Variables (can be set in .env file):")
fmt.Println(" CLIENT_ID - Client identifier (default: client1)")

1
go.mod
View File

@@ -21,6 +21,7 @@ require (
github.com/minio/md5-simd v1.1.2 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/pierrec/lz4/v4 v4.1.25 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rs/xid v1.6.0 // indirect

2
go.sum
View File

@@ -31,6 +31,8 @@ github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOF
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0=
github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=

54
internal/client/client.go
View File

@@ -4,7 +4,6 @@ package client
import (
"bytes"
"compress/gzip"
"encoding/json"
"fmt"
"io"
@@ -15,6 +14,7 @@ import (
"time"
"github.com/mistifyio/go-zfs"
"github.com/pierrec/lz4/v4"
)
// Client handles snapshot backup operations to a remote server.
@@ -110,7 +110,7 @@ func (c *Client) SendSnapshot(snapshot *zfs.Dataset) error {
}
// streamToS3 streams a ZFS snapshot to S3 storage via HTTP.
// The snapshot is optionally compressed with gzip before transmission.
// The snapshot is optionally compressed with LZ4 before transmission.
func (c *Client) streamToS3(snapshot *zfs.Dataset, uploadURL, storageKey string) error {
fmt.Printf("→ Streaming snapshot to S3...\n")
@@ -127,17 +127,18 @@ func (c *Client) streamToS3(snapshot *zfs.Dataset, uploadURL, storageKey string)
var reader io.Reader = zfsOut
// Apply gzip compression if enabled
// Apply LZ4 compression if enabled
if c.config.Compress {
fmt.Printf(" Compressing with gzip...\n")
fmt.Printf(" Compressing with LZ4...\n")
pr, pw := io.Pipe()
gzWriter := gzip.NewWriter(pw)
lz4Writer := lz4.NewWriter(pw)
lz4Writer.Apply(lz4.BlockSizeOption(lz4.BlockSize(4 * 1024 * 1024))) // 4MB blocks for better performance
go func() {
// Copy zfs output to gzip writer
io.Copy(gzWriter, zfsOut)
// Close gzip writer first to flush footer, then close pipe
gzWriter.Close()
// Copy zfs output to LZ4 writer
io.Copy(lz4Writer, zfsOut)
// Close LZ4 writer first to flush, then close pipe
lz4Writer.Close()
pw.Close()
}()
@@ -340,3 +341,38 @@ func (c *Client) GetRotationPolicy() (*ServerRotationPolicy, error) {
return &policyResp, nil
}
// ChangePassword changes the client's API key on the server.
// Requires the current API key for authentication and the new key.
func (c *Client) ChangePassword(newAPIKey string) error {
reqBody, _ := json.Marshal(map[string]string{
"client_id": c.config.ClientID,
"current_key": c.config.APIKey,
"new_key": newAPIKey,
})
resp, err := http.Post(c.config.ServerURL+"/client/change-password", "application/json", bytes.NewBuffer(reqBody))
if err != nil {
return fmt.Errorf("failed to change password: %v", err)
}
defer resp.Body.Close()
var result struct {
Success bool `json:"success"`
Message string `json:"message"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return fmt.Errorf("failed to decode response: %v", err)
}
if !result.Success {
return fmt.Errorf("failed to change password: %s", result.Message)
}
// Update local config with new key
c.config.APIKey = newAPIKey
fmt.Printf("✓ Password changed successfully\n")
return nil
}

2
internal/client/config.go
View File

@@ -19,7 +19,7 @@ type Config struct {
ServerURL string `json:"server_url"`
// LocalDataset is the ZFS dataset to backup
LocalDataset string `json:"local_dataset"`
// Compress enables gzip compression for transfers
// Compress enables LZ4 compression for transfers
Compress bool `json:"compress"`
}

17
internal/client/snapshot.go
View File

@@ -5,7 +5,6 @@ package client
import (
"bytes"
"compress/gzip"
"encoding/json"
"fmt"
"io"
@@ -17,6 +16,7 @@ import (
"time"
"github.com/mistifyio/go-zfs"
"github.com/pierrec/lz4/v4"
)
// SnapshotPolicy defines retention settings for automatic snapshots.
@@ -220,17 +220,18 @@ func (c *Client) streamIncrementalToS3(snapshot *zfs.Dataset, base, uploadURL, s
var reader io.Reader = zfsOut
// Apply gzip compression if enabled
// Apply LZ4 compression if enabled
if c.config.Compress {
fmt.Printf(" Compressing with gzip...\n")
fmt.Printf(" Compressing with LZ4...\n")
pr, pw := io.Pipe()
gzWriter := gzip.NewWriter(pw)
lz4Writer := lz4.NewWriter(pw)
lz4Writer.Apply(lz4.BlockSizeOption(lz4.BlockSize(4 * 1024 * 1024))) // 4MB blocks for better performance
go func() {
// Copy zfs output to gzip writer
io.Copy(gzWriter, zfsOut)
// Close gzip writer first to flush footer, then close pipe
gzWriter.Close()
// Copy zfs output to LZ4 writer
io.Copy(lz4Writer, zfsOut)
// Close LZ4 writer first to flush, then close pipe
lz4Writer.Close()
pw.Close()
}()

14
internal/restore/restore.go
View File

@@ -3,7 +3,6 @@
package restore
import (
"compress/gzip"
"encoding/json"
"fmt"
"io"
@@ -15,6 +14,7 @@ import (
"time"
"github.com/mistifyio/go-zfs"
"github.com/pierrec/lz4/v4"
)
// SnapshotMetadata represents snapshot information from the server.
@@ -97,7 +97,7 @@ func (c *Client) DisplaySnapshots(snapshots []*SnapshotMetadata) {
sizeGB := float64(snap.SizeBytes) / (1024 * 1024 * 1024)
compressed := ""
if snap.Compressed {
compressed = " (gz)"
compressed = " (lz4)"
}
fmt.Printf("%-4d %-25s %-20s %7.2f GB %-10s %s%s\n",
@@ -154,13 +154,9 @@ func (c *Client) RestoreSnapshot(snapshot *SnapshotMetadata, targetDataset strin
// Create decompression reader if needed
var reader io.Reader = resp.Body
if snapshot.Compressed {
fmt.Printf(" Decompressing...\n")
gzReader, err := gzip.NewReader(resp.Body)
if err != nil {
return fmt.Errorf("failed to create gzip reader: %v", err)
}
defer gzReader.Close()
reader = gzReader
fmt.Printf(" Decompressing with LZ4...\n")
lz4Reader := lz4.NewReader(resp.Body)
reader = lz4Reader
}
// Receive into ZFS

103
internal/server/admin_handlers.go
View File

@@ -683,3 +683,106 @@ func (s *Server) handleAdminUI(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.Write([]byte(adminPanelHTML))
}
// handleAdminResetClientPassword resets a client's API key to a new random value
func (s *Server) handleAdminResetClientPassword(w http.ResponseWriter, r *http.Request) {
admin, err := s.authenticateAdmin(r)
if err != nil || admin == nil {
http.Error(w, "Unauthorized", http.StatusUnauthorized)
return
}
if r.Method != http.MethodPost {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
clientID := r.URL.Query().Get("client_id")
if clientID == "" {
http.Error(w, "client_id required", http.StatusBadRequest)
return
}
// Get existing client
client, err := s.db.GetClient(clientID)
if err != nil {
http.Error(w, "Database error", http.StatusInternalServerError)
return
}
if client == nil {
http.Error(w, "Client not found", http.StatusNotFound)
return
}
// Generate new random API key
newAPIKey, err := generateToken()
if err != nil {
http.Error(w, "Failed to generate new API key", http.StatusInternalServerError)
return
}
// Update client with new API key
client.APIKey = hashAPIKey(newAPIKey)
if err := s.db.SaveClient(client); err != nil {
http.Error(w, "Failed to update client", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{
"success": true,
"message": "Client password reset successfully",
"api_key": newAPIKey, // Return the new key (only shown once!)
})
}
// handleClientChangePassword allows a client to change its own API key
func (s *Server) handleClientChangePassword(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
var req struct {
ClientID string `json:"client_id"`
CurrentKey string `json:"current_key"`
NewKey string `json:"new_key"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, "Invalid request body", http.StatusBadRequest)
return
}
if req.ClientID == "" || req.CurrentKey == "" || req.NewKey == "" {
http.Error(w, "client_id, current_key, and new_key required", http.StatusBadRequest)
return
}
// Authenticate with current key
if !s.authenticate(req.ClientID, req.CurrentKey) {
http.Error(w, "Invalid current API key", http.StatusUnauthorized)
return
}
// Get client
client, err := s.db.GetClient(req.ClientID)
if err != nil || client == nil {
http.Error(w, "Client not found", http.StatusNotFound)
return
}
// Update with new key
client.APIKey = hashAPIKey(req.NewKey)
if err := s.db.SaveClient(client); err != nil {
http.Error(w, "Failed to update password", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]interface{}{
"success": true,
"message": "Password changed successfully",
})
}

87
internal/server/admin_html.go
View File

@@ -339,6 +339,32 @@ const adminPanelHTML = `<!DOCTYPE html>
</div>
</div>
<!-- Client Change Password Modal -->
<div id="client-password-modal" class="modal-overlay hidden" onclick="closeModalOnOverlay(event, 'client-password-modal')">
<div class="modal-content" onclick="event.stopPropagation()">
<div class="modal-header">
<h3>Set Client API Key</h3>
<button type="button" class="close-btn" onclick="closeModal('client-password-modal')">&times;</button>
</div>
<form id="client-password-form">
<input type="hidden" id="client-password-client-id">
<div class="form-group">
<label>Client ID</label>
<input type="text" id="client-password-client-name" readonly>
</div>
<div class="form-group">
<label>New API Key</label>
<input type="text" id="client-password-new" required>
</div>
<div class="form-group">
<label>Confirm API Key</label>
<input type="text" id="client-password-confirm" required>
</div>
<button type="submit" class="btn btn-success" style="width: 100%; margin-top: 20px;">Set API Key</button>
</form>
</div>
</div>
<script>
let currentTab = 'clients';
@@ -432,6 +458,8 @@ const adminPanelHTML = `<!DOCTYPE html>
'<td>' + (c.enabled ? '<span class="badge badge-success">Enabled</span>' : '<span class="badge badge-danger">Disabled</span>') + '</td>' +
'<td>' +
'<button class="btn btn-sm btn-warning" onclick="editClient(\'' + c.client_id + '\')">Edit</button>' +
'<button class="btn btn-sm" style="background: #9b59b6; color: white;" onclick="showClientChangePassword(\'' + c.client_id + '\')">Set Key</button>' +
'<button class="btn btn-sm" style="background: #e67e22; color: white;" onclick="resetClientPassword(\'' + c.client_id + '\')">Reset Key</button>' +
'<button class="btn btn-sm btn-danger" onclick="deleteClient(\'' + c.client_id + '\')">Delete</button>' +
'</td>' +
'</tr>';
@@ -756,6 +784,65 @@ const adminPanelHTML = `<!DOCTYPE html>
}
}
// Reset client password/API key
async function resetClientPassword(clientId) {
if (!confirm('Reset API key for client "' + clientId + '"? The new key will be shown once.')) return;
try {
const res = await fetch('/admin/client/reset-password?client_id=' + clientId, { method: 'POST' });
const data = await res.json();
if (data.success) {
// Show the new API key in a prompt so it can be copied
prompt('New API key for ' + clientId + ' (copy this now - it won\\'t be shown again):', data.api_key);
} else {
alert(data.message || 'Failed to reset client password');
}
} catch (e) {
alert('Failed to reset client password');
}
}
// Show client password change modal
function showClientChangePassword(clientId) {
document.getElementById('client-password-client-id').value = clientId;
document.getElementById('client-password-client-name').value = clientId;
document.getElementById('client-password-new').value = '';
document.getElementById('client-password-confirm').value = '';
showModal('client-password-modal');
}
// Handle client password change form
document.getElementById('client-password-form').addEventListener('submit', async (e) => {
e.preventDefault();
const clientId = document.getElementById('client-password-client-id').value;
const newKey = document.getElementById('client-password-new').value;
const confirmKey = document.getElementById('client-password-confirm').value;
if (newKey !== confirmKey) {
alert('API keys do not match');
return;
}
try {
const res = await fetch('/admin/client/update', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ client_id: clientId, api_key: newKey })
});
const data = await res.json();
if (data.success) {
closeModal('client-password-modal');
alert('API key changed successfully for ' + clientId);
} else {
alert(data.message || 'Failed to change API key');
}
} catch (e) {
alert('Failed to change API key');
}
});
// Initialize
checkAuth();
</script>

4
internal/server/server.go
View File

@@ -214,7 +214,7 @@ func (s *Server) HandleUpload(w http.ResponseWriter, r *http.Request) {
// S3 upload
storageKey := fmt.Sprintf("%s/%s_%s.zfs", req.ClientID, req.DatasetName, timestamp)
if req.Compressed {
storageKey += ".gz"
storageKey += ".lz4"
}
respondJSON(w, http.StatusOK, UploadResponse{
@@ -507,6 +507,7 @@ func (s *Server) RegisterRoutes(mux *http.ServeMux) {
mux.HandleFunc("/download", s.HandleDownload)
mux.HandleFunc("/health", s.HandleHealth)
mux.HandleFunc("/rotation-policy", s.HandleRotationPolicy)
mux.HandleFunc("/client/change-password", s.handleClientChangePassword)
// Admin API routes
mux.HandleFunc("/admin/login", s.handleAdminLogin)
@@ -517,6 +518,7 @@ func (s *Server) RegisterRoutes(mux *http.ServeMux) {
mux.HandleFunc("/admin/client/create", s.handleAdminCreateClient)
mux.HandleFunc("/admin/client/update", s.handleAdminUpdateClient)
mux.HandleFunc("/admin/client/delete", s.handleAdminDeleteClient)
mux.HandleFunc("/admin/client/reset-password", s.handleAdminResetClientPassword)
mux.HandleFunc("/admin/snapshots", s.handleAdminGetSnapshots)
mux.HandleFunc("/admin/snapshot/delete", s.handleAdminDeleteSnapshot)
mux.HandleFunc("/admin/stats", s.handleAdminGetStats)