Skip to content

Commit 8c9d809

Browse files
authored
Merge pull request #13 from nullable-eth/fix-move-cleanup
fix: move cleanup before sync to free up space
2 parents a802e5b + 00ca93d commit 8c9d809

File tree

14 files changed

+333
-127
lines changed

14 files changed

+333
-127
lines changed

.github/workflows/release.yml

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -172,12 +172,18 @@ jobs:
172172

173173
- name: Build application
174174
run: |
175+
# Set build variables
176+
VERSION="${{ needs.check-changes.outputs.version }}"
177+
COMMIT="${{ github.sha }}"
178+
BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
179+
LDFLAGS="-X main.version=${VERSION} -X main.commit=${COMMIT} -X main.date=${BUILD_DATE}"
180+
175181
# Build for multiple architectures
176-
GOOS=linux GOARCH=amd64 go build -o syncarr-linux-amd64 ./cmd/syncarr
177-
GOOS=linux GOARCH=arm64 go build -o syncarr-linux-arm64 ./cmd/syncarr
178-
GOOS=windows GOARCH=amd64 go build -o syncarr-windows-amd64.exe ./cmd/syncarr
179-
GOOS=darwin GOARCH=amd64 go build -o syncarr-darwin-amd64 ./cmd/syncarr
180-
GOOS=darwin GOARCH=arm64 go build -o syncarr-darwin-arm64 ./cmd/syncarr
182+
GOOS=linux GOARCH=amd64 go build -ldflags "${LDFLAGS}" -o syncarr-linux-amd64 ./cmd/syncarr
183+
GOOS=linux GOARCH=arm64 go build -ldflags "${LDFLAGS}" -o syncarr-linux-arm64 ./cmd/syncarr
184+
GOOS=windows GOARCH=amd64 go build -ldflags "${LDFLAGS}" -o syncarr-windows-amd64.exe ./cmd/syncarr
185+
GOOS=darwin GOARCH=amd64 go build -ldflags "${LDFLAGS}" -o syncarr-darwin-amd64 ./cmd/syncarr
186+
GOOS=darwin GOARCH=arm64 go build -ldflags "${LDFLAGS}" -o syncarr-darwin-arm64 ./cmd/syncarr
181187
182188
- name: Upload build artifacts
183189
uses: actions/upload-artifact@v4
@@ -310,6 +316,10 @@ jobs:
310316
push: true
311317
tags: ${{ steps.meta.outputs.tags }}
312318
labels: ${{ steps.meta.outputs.labels }}
319+
build-args: |
320+
VERSION=${{ needs.check-changes.outputs.version }}
321+
COMMIT=${{ github.sha }}
322+
BUILD_DATE=${{ github.event.head_commit.timestamp }}
313323
cache-from: type=gha
314324
cache-to: type=gha,mode=max
315325

Dockerfile

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,32 @@ FROM golang:1.24-alpine AS builder
33

44
WORKDIR /app
55

6+
# Install git for version detection
7+
RUN apk add --no-cache git
8+
69
# Copy source code (includes go.mod)
710
COPY . .
811
RUN go mod download
912

10-
# Build the application
11-
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o syncarr ./cmd/syncarr
13+
# Build arguments for version information (can be provided by CI/CD)
14+
ARG VERSION
15+
ARG COMMIT
16+
ARG BUILD_DATE
17+
18+
# Auto-detect version information if not provided via build args
19+
RUN if [ -z "$VERSION" ]; then \
20+
VERSION=$(git describe --tags --always --dirty 2>/dev/null || echo "dev-local"); \
21+
fi && \
22+
if [ -z "$COMMIT" ]; then \
23+
COMMIT=$(git rev-parse --short HEAD 2>/dev/null || echo "local"); \
24+
fi && \
25+
if [ -z "$BUILD_DATE" ]; then \
26+
BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ"); \
27+
fi && \
28+
echo "Building with VERSION=$VERSION, COMMIT=$COMMIT, BUILD_DATE=$BUILD_DATE" && \
29+
CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo \
30+
-ldflags "-X main.version=$VERSION -X main.commit=$COMMIT -X main.date=$BUILD_DATE" \
31+
-o syncarr ./cmd/syncarr
1232

1333
# Runtime stage
1434
FROM alpine:latest

README.md

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ services:
7979

8080
- **🏷️ Label-based Sync**: Automatically sync only media items with specific Plex labels
8181
- **⚡ High-Performance Transfers**: Uses rsync for fast, resumable file transfers
82-
- **🔄 6-Phase Sync Process**: Content discovery → File transfer → Library refresh → Content matching → Metadata sync → Cleanup
82+
- **🔄 7-Phase Sync Process**: Content discovery → Cleanup → File transfer → Library refresh → Content matching → Metadata sync
8383
- **📊 Comprehensive Metadata Sync**: Titles, summaries, ratings, genres, labels, collections, artwork, and more
8484
- **👁️ Watched State Sync**: Keep viewing progress synchronized between servers
8585
- **🔄 Incremental Updates**: Only transfer changed or new content
@@ -103,7 +103,6 @@ services:
103103
- **🐳 Docker Ready**: Containerized application with health checks
104104
- **📝 Structured Logging**: JSON logging with configurable levels (DEBUG, INFO, WARN, ERROR)
105105
- **🔄 Continuous & One-shot Modes**: Run continuously or execute single sync cycles
106-
- **🎛️ Force Full Sync**: Bypass incremental checks for complete re-synchronization
107106
- **📈 Performance Monitoring**: Detailed transfer statistics and timing information
108107
- **🔍 Content Matching**: Intelligent filename-based matching between source and destination
109108

@@ -146,7 +145,6 @@ services:
146145
| `SYNC_INTERVAL` | Minutes between sync cycles | `60` | ❌ |
147146
| `LOG_LEVEL` | Logging level | `INFO` | ❌ |
148147
| `DRY_RUN` | Test mode without changes | `false` | ❌ |
149-
| `FORCE_FULL_SYNC` | Force complete sync | `false` | ❌ |
150148

151149
### Path Mapping
152150

@@ -228,9 +226,6 @@ docker run --rm -v $(pwd)/config:/config syncarr --oneshot
228226
# Validate configuration
229227
docker run --rm -v $(pwd)/config:/config syncarr --validate
230228
231-
# Force full synchronization (bypasses incremental checks)
232-
docker run --rm -v $(pwd)/config:/config syncarr --force-full-sync --oneshot
233-
234229
# Show version information
235230
docker run --rm syncarr --version
236231
@@ -278,14 +273,14 @@ docker-compose exec syncarr ./syncarr --validate
278273
## 🏗️ Architecture
279274

280275
<details>
281-
<summary><strong>📊 6-Phase Sync Process</strong></summary>
276+
<summary><strong>📊 7-Phase Sync Process</strong></summary>
282277

283278
1. **🔍 Content Discovery**: Scan source Plex server for labeled media
284-
2. **📂 File Transfer**: Copy media files using high-performance rsync
285-
3. **🔄 Library Refresh**: Update destination Plex library
286-
4. **🎯 Content Matching**: Match source items to destination items by filename
287-
5. **📝 Metadata Sync**: Synchronize comprehensive metadata between matched items
288-
6. **🧹 Cleanup**: Remove orphaned files and update statistics
279+
2. **🧹 Cleanup**: Remove orphaned files from destination (frees space and ensures Plex detects removals)
280+
3. **📂 File Transfer**: Copy media files using high-performance transfers
281+
4. **🔄 Library Refresh**: Update destination Plex library
282+
5. **🎯 Content Matching**: Match source items to destination items by filename
283+
6. **📝 Metadata Sync**: Synchronize comprehensive metadata between matched items
289284

290285
</details>
291286

cmd/syncarr/main.go

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,9 @@ var (
2222
func main() {
2323
// Command line flags
2424
var (
25-
showVersion = flag.Bool("version", false, "Show version information")
26-
validateOnly = flag.Bool("validate", false, "Validate configuration and exit")
27-
oneShot = flag.Bool("oneshot", false, "Run sync once and exit (don't run continuously)")
28-
forceFullSync = flag.Bool("force-full-sync", false, "Force a complete synchronization, bypassing incremental checks")
25+
showVersion = flag.Bool("version", false, "Show version information")
26+
validateOnly = flag.Bool("validate", false, "Validate configuration and exit")
27+
oneShot = flag.Bool("oneshot", false, "Run sync once and exit (don't run continuously)")
2928
)
3029
flag.Parse()
3130

@@ -40,11 +39,6 @@ func main() {
4039
log.Fatalf("Failed to load configuration: %v", err)
4140
}
4241

43-
// Override force full sync if specified via command line
44-
if *forceFullSync {
45-
cfg.ForceFullSync = true
46-
}
47-
4842
// Validate configuration
4943
if err := cfg.Validate(); err != nil {
5044
log.Fatalf("Configuration validation failed: %v", err)
@@ -65,7 +59,6 @@ func main() {
6559
"source_host": cfg.Source.Host,
6660
"destination_host": cfg.Destination.Host,
6761
"sync_label": cfg.SyncLabel,
68-
"force_full_sync": cfg.ForceFullSync,
6962
"dry_run": cfg.DryRun,
7063
}).Info("SyncArr starting up")
7164

@@ -80,11 +73,6 @@ func main() {
8073
}
8174
}()
8275

83-
// Handle force full sync
84-
if err := sync.HandleForceFullSync(); err != nil {
85-
log.WithError(err).Fatal("Failed to handle force full sync")
86-
}
87-
8876
// Set up signal handling for graceful shutdown
8977
sigChan := make(chan os.Signal, 1)
9078
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

internal/config/config.go

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ type Config struct {
2222
SSH SSHConfig `json:"ssh"`
2323
Performance PerformanceConfig `json:"performance"`
2424
Transfer TransferConfig `json:"transfer"`
25-
ForceFullSync bool `json:"forceFullSync"`
2625
DryRun bool `json:"dryRun"`
2726
LogLevel string `json:"logLevel"`
2827
}
@@ -91,9 +90,8 @@ func LoadConfig() (*Config, error) {
9190
Port: getEnvWithDefault("SSH_PORT", "22"),
9291
KeyPath: getEnvWithDefault("SSH_KEY_PATH", ""), // Keep for future use
9392
},
94-
DryRun: parseBoolEnv("DRY_RUN", false),
95-
LogLevel: getEnvWithDefault("LOG_LEVEL", "INFO"),
96-
ForceFullSync: parseBoolEnv("FORCE_FULL_SYNC", false),
93+
DryRun: parseBoolEnv("DRY_RUN", false),
94+
LogLevel: getEnvWithDefault("LOG_LEVEL", "INFO"),
9795
}
9896

9997
// Set protocol based on RequireHTTPS

internal/config/config_test.go

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ func TestLoadConfig(t *testing.T) {
2424
"DEST_ROOT_DIR": "/test/dest",
2525
"LOG_LEVEL": "DEBUG",
2626
"DRY_RUN": "true",
27-
"FORCE_FULL_SYNC": "false",
2827
}
2928

3029
// Set environment variables
@@ -77,10 +76,6 @@ func TestLoadConfig(t *testing.T) {
7776
t.Error("Expected DryRun to be true")
7877
}
7978

80-
if cfg.ForceFullSync {
81-
t.Error("Expected ForceFullSync to be false")
82-
}
83-
8479
// Test log level
8580
if cfg.LogLevel != "DEBUG" {
8681
t.Errorf("Expected log level 'DEBUG', got '%s'", cfg.LogLevel)

internal/discovery/content_matcher.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88
"github.com/nullable-eth/syncarr/internal/plex"
99
)
1010

11-
// ContentMatcher handles Phase 5: Content Matching
11+
// ContentMatcher handles Phase 6: Content Matching
1212
type ContentMatcher struct {
1313
sourceClient *plex.Client
1414
destClient *plex.Client
@@ -31,9 +31,9 @@ func NewContentMatcher(sourceClient, destClient *plex.Client, log *logger.Logger
3131
}
3232
}
3333

34-
// MatchItemsByFilename implements Phase 5: Content Matching by filename with full metadata
34+
// MatchItemsByFilename implements Phase 6: Content Matching by filename with full metadata
3535
func (cm *ContentMatcher) MatchItemsByFilename(sourceItems []*EnhancedMediaItem) ([]ItemMatch, error) {
36-
cm.logger.Info("Phase 5: Starting enhanced content matching by filename with full metadata loading")
36+
cm.logger.Info("Phase 6: START - Content Matching")
3737

3838
// Get all items from destination server and load their full metadata
3939
destLibraries, err := cm.destClient.GetLibraries()

internal/discovery/discovery.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ func NewContentDiscovery(sourceClient *plex.Client, syncLabel string, logger *lo
3535
// 2. If any movie contains the sync tag, add it to the processing list with complete metadata
3636
// If any TV show contains the sync label, list all episodes of all seasons and add them with complete metadata
3737
func (cd *ContentDiscovery) DiscoverSyncableContent() ([]*EnhancedMediaItem, error) {
38-
cd.logger.Info("Phase 1: Starting enhanced content discovery with full metadata loading")
38+
cd.logger.Debug("Phase 1: Starting enhanced content discovery with full metadata loading")
3939

4040
var itemsToSync []*EnhancedMediaItem
4141

@@ -45,7 +45,7 @@ func (cd *ContentDiscovery) DiscoverSyncableContent() ([]*EnhancedMediaItem, err
4545
return nil, fmt.Errorf("failed to get libraries: %w", err)
4646
}
4747

48-
cd.logger.WithField("library_count", len(libraries)).Info("Retrieved libraries from source server")
48+
cd.logger.WithField("library_count", len(libraries)).Debug("Retrieved libraries from source server")
4949

5050
for _, library := range libraries {
5151
cd.logger.WithFields(map[string]interface{}{
@@ -67,7 +67,7 @@ func (cd *ContentDiscovery) DiscoverSyncableContent() ([]*EnhancedMediaItem, err
6767
"library_id": library.Key,
6868
"sync_label": cd.syncLabel,
6969
"labeled_items": len(labeledItems),
70-
}).Info("Retrieved items with sync label, now loading full metadata")
70+
}).Debug("Retrieved items with sync label, now loading full metadata")
7171

7272
for i, item := range labeledItems {
7373
cd.logger.WithFields(map[string]interface{}{
@@ -92,7 +92,7 @@ func (cd *ContentDiscovery) DiscoverSyncableContent() ([]*EnhancedMediaItem, err
9292
}
9393
}
9494

95-
cd.logger.WithField("total_items_to_sync", len(itemsToSync)).Info("Phase 1 & 2: Enhanced content discovery with full metadata complete")
95+
cd.logger.WithField("total_items_to_sync", len(itemsToSync)).Debug("Phase 1 and 2: Enhanced content discovery with full metadata complete")
9696

9797
return itemsToSync, nil
9898
}

internal/discovery/library_manager.go

Lines changed: 92 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88
"github.com/nullable-eth/syncarr/internal/plex"
99
)
1010

11-
// LibraryManager handles Phase 4: Library refresh and monitoring
11+
// LibraryManager handles Phase 5: Library refresh and monitoring
1212
type LibraryManager struct {
1313
destClient *plex.Client
1414
logger *logger.Logger
@@ -24,7 +24,7 @@ func NewLibraryManager(destClient *plex.Client, log *logger.Logger) *LibraryMana
2424

2525
// TriggerRefreshAndWait triggers library scans and waits for completion
2626
func (lm *LibraryManager) TriggerRefreshAndWait() error {
27-
lm.logger.Info("Phase 4: Triggering library refresh on destination server")
27+
lm.logger.Info("Phase 5: START - Library Refresh")
2828

2929
// First, wait for any existing scans to complete before starting new ones
3030
lm.logger.Debug("Checking for existing library scans before starting new ones")
@@ -81,7 +81,51 @@ func (lm *LibraryManager) TriggerRefreshAndWait() error {
8181
}
8282

8383
// Monitor scan completion for successfully triggered scans
84-
return lm.waitForAllScansComplete(successfulScans)
84+
if err := lm.waitForAllScansComplete(successfulScans); err != nil {
85+
return fmt.Errorf("library scan failed: %w", err)
86+
}
87+
88+
lm.logger.Info("Library scans completed, now triggering metadata refresh for all libraries")
89+
90+
// Trigger metadata refresh for all libraries to populate initial metadata
91+
var successfulMetadataRefresh []plex.Library
92+
var failedMetadataRefresh []string
93+
94+
for _, library := range libraries {
95+
lm.logger.WithFields(map[string]interface{}{
96+
"library_id": library.Key,
97+
"library_name": library.Title,
98+
}).Debug("Triggering metadata refresh")
99+
100+
if err := lm.destClient.TriggerMetadataRefresh(library.Key); err != nil {
101+
lm.logger.WithError(err).WithFields(map[string]interface{}{
102+
"library_id": library.Key,
103+
"library_name": library.Title,
104+
}).Error("Failed to trigger metadata refresh")
105+
failedMetadataRefresh = append(failedMetadataRefresh, fmt.Sprintf("%s (%s)", library.Title, library.Key))
106+
} else {
107+
successfulMetadataRefresh = append(successfulMetadataRefresh, library)
108+
lm.logger.WithFields(map[string]interface{}{
109+
"library_id": library.Key,
110+
"library_name": library.Title,
111+
}).Debug("Successfully triggered metadata refresh")
112+
}
113+
}
114+
115+
// Log metadata refresh trigger results
116+
if len(failedMetadataRefresh) > 0 {
117+
lm.logger.WithField("failed_libraries", failedMetadataRefresh).Warn("Some metadata refreshes failed to trigger")
118+
}
119+
120+
// If no metadata refreshes were successfully triggered, don't wait
121+
if len(successfulMetadataRefresh) == 0 {
122+
return fmt.Errorf("failed to trigger any metadata refreshes")
123+
}
124+
125+
lm.logger.WithField("library_count", len(successfulMetadataRefresh)).Info("Waiting for metadata refresh to complete")
126+
127+
// Monitor metadata refresh completion
128+
return lm.waitForAllMetadataRefreshComplete(successfulMetadataRefresh)
85129
}
86130

87131
// waitForExistingScansComplete waits for any existing library scans to complete
@@ -211,3 +255,48 @@ func (lm *LibraryManager) logScanProgress(activities []plex.Activity) {
211255
lm.logger.WithFields(fields).Debug("Scan progress")
212256
}
213257
}
258+
259+
// waitForAllMetadataRefreshComplete waits for all metadata refreshes to complete
260+
func (lm *LibraryManager) waitForAllMetadataRefreshComplete(libraries []plex.Library) error {
261+
lm.logger.WithField("library_count", len(libraries)).Info("Monitoring metadata refresh completion")
262+
263+
const maxWaitTime = 30 * time.Minute // Metadata refresh can take longer than scans
264+
const checkInterval = 15 * time.Second
265+
startTime := time.Now()
266+
267+
for {
268+
if time.Since(startTime) > maxWaitTime {
269+
lm.logger.Warn("Metadata refresh wait timeout reached, proceeding anyway")
270+
return nil
271+
}
272+
273+
// Check if any metadata refresh activities are still running
274+
metadataInProgress, activities, err := lm.destClient.IsLibraryScanInProgress()
275+
if err != nil {
276+
lm.logger.WithError(err).Warn("Error checking metadata refresh status")
277+
return nil
278+
}
279+
280+
if !metadataInProgress {
281+
lm.logger.Info("All metadata refreshes completed successfully")
282+
return nil
283+
}
284+
285+
// Log progress for metadata refresh activities
286+
var refreshActivities []string
287+
for _, activity := range activities {
288+
if activity.Type == "library.refresh" {
289+
refreshActivities = append(refreshActivities, activity.Title)
290+
}
291+
}
292+
293+
if len(refreshActivities) > 0 {
294+
lm.logger.WithFields(map[string]interface{}{
295+
"active_refreshes": refreshActivities,
296+
"elapsed_time": time.Since(startTime).Round(time.Second),
297+
}).Debug("Metadata refresh still in progress")
298+
}
299+
300+
time.Sleep(checkInterval)
301+
}
302+
}

0 commit comments

Comments
 (0)