#!/usr/bin/perl -w # Copyright © 2011-2013 Jamie Zawinski # # Permission to use, copy, modify, distribute, and sell this software and its # documentation for any purpose is hereby granted without fee, provided that # the above copyright notice appear in all copies and that both that # copyright notice and this permission notice appear in supporting # documentation. No representations are made about the suitability of this # software for any purpose. It is provided "as is" without express or # implied warranty. # # For doing a partial one-way sync of iTunes libraries, using rsync, ssh # and AppleScript. # # Usage: # # itunes-rsync.pl -v 'Local Playlist Name' remote-host-name # # It does the following: # # - Get a list of the tracks in the local iTunes playlist. # # - Run rsync to push those files to the remote machine, deleting any files # that are on the remote machine but are not in the local playlist. # # - Get the local iTunes metadata about the changed tracks via AppleScript. # # - Import newly-added files into the remote iTunes. # # - Update the remote metadata of any changed files to match the local # metadata. # # Updating stars: # # With the --stars-only option, it will update the star ratings and # checkedness of *every* track. This will take a long time, but might # sometimes be necessary, since changing the rating of a track in # iTunes doesn't change the file's write date, so there's no way to # tell when that change has happened. # # Comparing: # # The --diff option will compare local and remote metadata. It's slow. # Like, 40+ minutes slow. And sometimes it will time out and claim there # are no differences. # # Caveats: # # - Error recovery is basically nonexistent. # # - If you make any changes to the name of an artist or track, including # capitalization, it will delete and re-transfer the files (since rsync # has no concept of case-insensitive file systems). # # - The "Date Added" attribute in the remote iTunes will be nonsense. # # - This script runs rsync and ssh four times total, so to avoid typing # your password over and over, set up ssh keys/agents for passwordless # login. # # - It assumes that both machines have their iTunes libraries in the # default place. (Edit $music_dir, or use symlinks.) # # Created: 2-May-2011. require 5; use diagnostics; use strict; use File::Path ('make_path'); my $progname = $0; $progname =~ s@.*/@@g; my $version = q{ $Revision: 1.25 $ }; $version =~ s/^[^\d]+([\d.]+).*/$1/; my $verbose = 1; my $debug_p = 0; # Surely there is some way to ask iTunes what this path is via AppleScript, # but I have not found it. my $music_dir = 'Music/iTunes/iTunes Media/Music/'; # Update: I guess this would work, but what a fragile crock of shit! # # lib=`defaults read com.apple.iapps iTunesRecentDatabases | # sed -n 's@^.*"file://localhost\(/.*\)".*$@\1@p' | # sed 's/%20/ /g' | head -1` # dir=`cat "$lib" | # sed -n s'@^.*>Music Folder<.*>file://localhost\(/[^<>]*/\).*@\1Music/@p' | # sed 's/%20/ /g' | head -1` my @all_fields = ('name', 'artist', 'album artist', 'album', 'composer', 'comment', 'genre', 'year', 'track number', 'track count', 'bpm', 'compilation', 'volume adjustment', 'video kind', 'enabled', 'rating', 'start', 'finish', 'bookmarkable', 'shufflable'); my $forever = (60 * 60 * 8) . " seconds"; # Talks to the local iTunes and returns the contents of a playlist # as a list of the underlying file names (absolute paths). # sub get_playlist($) { my ($playlist) = @_; print STDERR "$progname: getting playlist \"$playlist\"" . " from local iTunes...\n" if ($verbose); my $script = ("with timeout of $forever\n" . " tell application \"iTunes\"\n" . " set L to the location of every file track" . " of playlist \"$playlist\"\n" . " end tell\n" . " set OUTPUT to \"\"\n" . " repeat with P in L\n" . " set OUTPUT to OUTPUT & (the POSIX path of P) & \"\\n\"\n" . " end repeat\n" . "end timeout\n" . "OUTPUT\n"); my $cmd = "osascript -e '$script'"; my @lines = `$cmd`; my @L; foreach (@lines) { chomp $_; next unless $_; push @L, $_; } print STDERR "$progname: " . ($#L+1) . " tracks.\n" if ($verbose); return @L; } my $rm_minus_rf = undef; END { system ("rm", "-rf", $rm_minus_rf) if $rm_minus_rf; } # Creates a directory tree populated with symbolic links to the # given files. Returns the name of that tmp directory. # sub link_farm(@) { my (@files) = @_; my $dir = sprintf ("%s/itrs.%08x", ($ENV{TMPDIR} ? $ENV{TMPDIR} : "/tmp"), rand(0xFFFFFFFF)); $dir =~ s@//+@/@gs; $rm_minus_rf = $dir; # nuke it even at abnormal exits. system ("rm", "-rf", $rm_minus_rf); print STDERR "$progname: creating link farm: $dir\n" if ($verbose > 1); umask (022); foreach my $ofile (@files) { my $file = $ofile; $file =~ s@^/Users/[^/]+/$music_dir@@si; my ($dir2) = ($file =~ m@^(.*)/[^/]+$@si); error ("unparsable: $file") unless $dir2; my $td = "$dir/$dir2"; make_path ($td); symlink ($ofile, "$dir/$file") || error ("ln -s '$ofile' '$dir/$file': $!"); } return $dir; } # Given a remote host (or "user@host"), runs rsync to push the given # iTunes files into the remote iTunes music directory, then updates # the database inside the remote iTunes. # sub sync_files($$$$) { my ($host, $stars_only, $force_p, $playlist) = @_; my @files = get_playlist ($playlist); my $link_farm = link_farm (@files); my $md2 = $music_dir; $md2 =~ s@([^-_.,a-z\d/])@\\$1@gsi; $link_farm .= '/' unless ($link_farm =~ m@/$@s); $md2 .= '/' unless ($md2 =~ m@/$@s); # Ok, get this shit. We have to run rsync three times. # # 1: "rsync --dry-run" to compute the list of changed files. # Without that, we don't know which metadata to extract from # the local iTunes and transmit to the remote. # # 2: "rsync" to actually transfer the files. # # 2b: Then we tell the remote iTunes to update the metadata. # Oops, a side-effect of this is that iTunes writes to the # files! So the write dates change... AND SO DO THE CHECKSUMS. # I have no idea why the sums change, but they do! # # 3: So after updating the metadata, we rsync again, so that the # remote iTunes database has the right data; and the files have # the same data on both machines. This is the *same* data that # is in the iTunes DB, but as-written-by local iTunes instead of # as-written-by remote iTunes. # # What a steaming load of shit. # # Oh, I also tried simply not updating the metadata in the remote iTunes # if it hadn't changed, but that didn't work either, possibly due to # rounding errors? Possibly due to other AppleScript idiocy? You read # "volume adjustment = 48" on local host, write "48" into remote host, # read it back, and now it's 49 instead. Who the fuck knows. # # It would be possible to combine steps 1 and 2a (just do the first sync # without a --dry-run first, and parse the output of that rsync) but then # we couldn't issue the warning, "lots of files to be deleted, use # --force if you're sure" warning, # # Fuck AppleScript and the iTunes it rode in on. my @cmd = ("rsync"); push @cmd, ("-aOL", "--delete-during", "$link_farm", "$host:$md2"); my %files; foreach my $f (@files) { my $f2 = $f; $f2 =~ s@^/Users/[^/]+/$music_dir@@si; $files{$f2} = 1; } ############################################################# Pass 1! print STDERR "$progname: getting list of changed files...\n" if ($verbose); my @cmd2 = @cmd; push @cmd2, ("-v", "--dry-run"); my $cmd2 = ''; foreach (@cmd2) { $_ = "\"$_\"" if m/ /s; $cmd2 .= " $_"; } print STDERR "$progname: $cmd2\n" if ($verbose > 1); my $result = `( $cmd2 ) 2>&1`; my @changes = (); error ($result) if ($result =~ m/rsync error: unexplained error/si); # #### To manually force an update of the metadata of particular files, # put 'em here. Or just "touch" them. # $result .= ''; if ($verbose > 2) { my $s = $result; $s =~ s/^/\t/gm; print STDERR "$s\n"; } my @deleting = (); foreach my $line (split (/[\r\n]+/, $result)) { next unless $line; next if ($line =~ m/^building file list|^sending incremental file list/s); next if ($line =~ m/^sent \d+ bytes|^total size is \d+ /s); next if ($line =~ m/^Killed by signal 1\./s); next if ($line =~ m@/$@s); if ($line =~ m/^deleting +(.*)/s) { push @deleting, $1; next; } $line =~ s@[\\][#](\d\d\d)@{ chr(oct($1)) }@gsex; # Unicrud if ($files{$line}) { push @changes, $line; } else { print STDERR "$progname: not a file: $line\n"; } } my $n = $#deleting+1; if ($n >= 10 && !$force_p) { print STDERR "$progname: $n files to be deleted!" . " Use --force if you're sure!\n\n"; foreach my $d (sort @deleting) { print STDERR "\t$d\n"; } print STDERR "\n"; exit 1; } my $nchanges = $#changes+1 + $n; print STDERR "$progname: $nchanges changed files.\n" if ($verbose); if ($nchanges < 0 && !$debug_p && !$stars_only) { exit(0); } ############################################################# Pass 2! push @cmd, ($verbose ? "-vhi" : "-q"); push @cmd, "--dry-run" if ($debug_p); if ($debug_p) { print STDERR "$progname: not running: " . join(' ', @cmd) . "\n"; } elsif ($nchanges > 0) { print STDERR "$progname: " . join(' ', @cmd) . "\n" if ($verbose > 1); system (@cmd); } ############################################################# Pass 2B! @changes = sort (keys (%files)) if ($stars_only); # do them all. my $metadata = sync_metadata ($host, $stars_only, @changes); ############################################################# Pass 3! print STDERR "$progname: metadata updated, syncing files again!\n" if ($verbose); if ($debug_p) { print STDERR "$progname: not running: " . join(' ', @cmd) . "\n"; } else { print STDERR "$progname: " . join(' ', @cmd) . "\n" if ($verbose > 1); system (@cmd); } print STDERR "\n$progname: done! $nchanges files updated.\n\n" if ($verbose); } # Given some AppleScript code, runs it on the remote host, via ssh. # sub run_remote_applescript($$) { my ($host, $script) = @_; my $remote = "ssh -xTC $host osascript"; if ($debug_p) { print STDERR "$progname: not running: $remote\n"; } else { print STDERR "$progname: updating metadata on remote iTunes...\n" if ($verbose); } if ($verbose > 1) { print STDERR "$progname: $remote\n"; my $s = $script; $s =~ s/^/\t/gm; print STDERR "$s\n"; } if (! $debug_p) { open (my $pipe, '|-', $remote) || error ("open: $remote: $!"); (print $pipe $script) || error ("pipe: $!"); close $pipe; } } # Copies iTunes metadata from the local iTunes to the remote iTunes, # for the list of (local, absolute path) files. # # Returns a hash of the local metadata interrogated. # sub sync_metadata($$@) { my ($host, $stars_only, @files) = @_; my $remote_applescript = ''; my @read_fields = @all_fields; my @write_fields = ($stars_only ? ('name', 'artist', 'enabled', 'rating', 'video kind') : @read_fields); # @write_fields = @read_fields; my %result; my $script = ("set LF to \"\\n\"\n" . "set OUTPUT to \"\"\n" . "with timeout of $forever\n" . " tell application \"iTunes\"\n" . " set PF to POSIX path of" . " (path to home folder from user domain) &" . " \"$music_dir\"\n\n" . " repeat with P in {"); foreach my $f (@files) { my $f2 = $f; $f2 =~ s/"/\\"/gs; $script .= "PF & "; $script .= "\"$f2\", "; } $script =~ s/, *$//s; $script .= "}\n"; $script .= (" try\n" . " set P to P as POSIX file as alias\n" . " set T to (add P as alias)\n" . " tell T\n" . " set OUTPUT to OUTPUT &" . " (POSIX path of P) & \"\t\" & " . join (" & \"\t\" & ", @read_fields) . " & LF\n" . " end tell\n" . " on error errmesg number errn\n" . " set OUTPUT to OUTPUT & \"ERROR: \" & errmesg" . " & \": \" & (errn as text) & LF\n" . " end try\n\n" . " end repeat\n" . " end tell\n" . "end timeout\n" . "OUTPUT"); $script =~ s/([\"\$])/\\$1/gs; print STDERR "$progname: getting metadata from local iTunes...\n" if ($verbose); my $cmd = "osascript -e \"$script\""; my $result = `$cmd`; print STDERR "$progname: result:\n\n$result\n\n" if ($verbose > 3); # Generate the script to run on the other end. # If we're adding a new track, we need to use "add" on the path. # If we're updating an old track, we need to find the "file track" # that corresponds to its file name. Guess what? There's no way # to tell whether a file path is already present in the library, or # to find the "file track" that corresponds to a file, without # iterating the entire library and comparing them one at a time, # which is really slow. # # The only other way to accomplish is to "add" on the existing file, # which returns a "file track". But this *sometimes* (but not always) # has the side effect of adding a duplicate entry, resulting in two # entries that have the same "path" and other fields but different # "database ID"s. So after doing this, we have to find and delete # those dups, too. # This code builds a function to update each track rather than just # inlining the code for two reasons: first, it makes the script smaller; # but second, it allows us to see status updates in realtime instead of # only getting them all at the end. The AppleScript "log" function # is only implemented in "AppleScript Editor.app" and in "osascript". # If you call "log" while inside of "tell application", it's a no-op. # So the only way to get real-time output to STDERR to show up is to # have a single "tell application" for each track we're updating, and # call "log" outside of that. (This may actually slow things down a # little bit, but the trade-off is worth it.) my $timeout = $forever; $timeout = 60 * 2 . " seconds"; $script = ("to W(arg_pathname"); foreach my $k (@write_fields) { my $k2 = $k; $k2 =~ s/ /_/gs; $script .= ", arg_$k2"; } $script .= (")\n" . " set LF to \"\\n\"\n" . " set PF to POSIX path of" . " (path to home folder from user domain) & \"$music_dir\"\n" . "\n" . " set OUTPUT to \"\"\n" . " with timeout of $timeout\n" . " set P to POSIX file (PF & arg_pathname)\n" . " tell application \"iTunes\"\n"); # Fucking pseudo-enums only parse within the application's context!! foreach my $f ("music video", "movie", "tv show", "podcast") { my $a = 'arg_video_kind'; $script .= " if $a is \"$f\" then set $a to $f\n"; } $script .= (" try\n" . " set T to (add P as alias)\n" . " tell T\n"); foreach my $k (@write_fields) { my $k2 = $k; my $v = "arg_$k"; $v =~ s/ /_/gs; if ($k =~ m/^volume adjustment/si) { # WTF? Seriously, WTF? $k2 = "{$k2}"; $v = "{$v}"; } $script .= " set $k2 to $v\n"; } $script .= (" set OUTPUT to OUTPUT &" . " \"Updated: \" & artist & \" - \" & name\n" . " end tell\n"); # Iterate every track with the same artist/name as this # one, and see if any of them have the same pathname but # a different database ID. Nuke those. # # We have to iterate tracks, build a list of IDs, then # iterate that because Applescript is such a steaming # pile that you can't delete items that are members of a # list that you are iterating! # $script .= (" set TID to database ID of T\n" . " set TP to location of T\n" . " set TIDS to {}\n" . " repeat with T2 in every file track whose" . " artist is arg_artist" . " and name is arg_name\n" . " set TID2 to database ID of T2\n" . " if TID is not TID2" . " and TP is (location of T2) then\n" . " set TIDS to TIDS & TID2\n" . " end if\n" . " end repeat\n" . " repeat with TID2 in TIDS\n" . " set T2 to (first file track whose" . " database ID is TID2)\n"); $script .= (" set OUTPUT to OUTPUT & LF & \"Deleted dup: \"" . " & TID & \" \" & TID2\n") if ($verbose > 1); $script .= (" delete T2\n" . " end repeat\n" . " on error errmesg number errn\n" . " set OUTPUT to OUTPUT & LF & \"ERROR: \" & errmesg" . " & \": \" & (errn as text)\n" . " end try\n" . " end tell\n" . " end timeout\n"); $script .= (" log OUTPUT\n") if ($verbose); $script .= ("end W\n\n"); foreach my $line (split (/[\r\n]+/, $result)) { if ($line =~ m/^ERROR:/si) { print STDERR "$progname: $line\n"; next; } my @vv = split (/\t/, $line); my $path = shift @vv; my %ff; my @kk = @read_fields; while (@vv) { my $k = shift @kk; my $v = shift @vv; error ("unparsable line: $line\n") unless ($k && defined($v)); $v =~ s/\"/\\"/gs; $ff{$k} = $v; } $result{$path} = \%ff; $path =~ s@^/Users/[^/]+/$music_dir@@si; $script .= "W(\"" . $path . "\""; foreach my $k (@write_fields) { my $v = $ff{$k} || ''; $v = "\"$v\"" unless ($v =~ m/^(true|false|-?\d+\.?\d*)$/s); $v =~ s/^(-?\d+\.\d{3})\d+$/$1/s; # 3 decimals $script .= ", $v"; } $script .= ")\n"; } # Also remove dead tracks. I'm not sure why this is necessary, but # sometimes file tracks are getting left behind for files that no # longer exist. $script .= ("log \"Cleaning up...\"\n") if ($verbose); $script .= ("\n" . "set OUTPUT to \"\"\n" . "with timeout of $forever\n" . " tell application \"iTunes\"\n" . " set LIB to library playlist 1\n" . " repeat with TN from (count of file tracks of LIB)" . " to 1 by -1\n" . " set T to file track TN of LIB\n" . " if location of T is missing value then\n" . " tell T\n" . " set OUTPUT to OUTPUT & \"Deleted: \"" . " & artist & \" - \" & name & \"\\n\"\n" . " end tell\n" . " delete T\n" . " end if\n" . " end repeat\n" . " end tell\n" . "end timeout\n" . "OUTPUT\n"); $script .= "log \"Remote iTunes metadata updated.\"\n" if ($verbose); run_remote_applescript ($host, $script); return \%result; } # Compare the metadata of the local playlist with the remote library. # sub diff_files($$) { my ($host, $playlist) = @_; my $timeout = $forever; $timeout = (60 * 60) . " seconds"; my @read_fields = @all_fields; my $script = ("set OUTPUT to \"\"\n" . "tell application \"iTunes\"\n" . " repeat with T in every file track %%PL%%\n" . " tell T\n" . " set OUTPUT to OUTPUT &" . " (POSIX path of (location of T as text))" . " & \"\\t\" & " . join (" & \"\\t\" & ", @read_fields) . " & \"\\n\"\n" . " end tell\n" . " end repeat\n" . "end tell\n" . "OUTPUT"); $script =~ s/ +/ /gm; my ($script1, $script2) = ($script, $script); $script1 =~ s/%%PL%%/of playlist "$playlist"/s; $script2 =~ s/%%PL%%/of playlist "Music"/s; $script1 =~ s/([\"\$])/\\$1/gs; print STDERR "$progname: getting metadata from local iTunes...\n" if ($verbose); my $cmd = "osascript -e \"$script1\""; my $result1 = `$cmd`; print STDERR "$progname: getting metadata from remote iTunes...\n" if ($verbose); $cmd = "ssh -xTC $host /bin/sh -c " . "'osascript << __EOF__\n$script2\n__EOF__\n'"; my $result2 = `$cmd`; my %all_files; my $R = sub($$$) { my ($type, $table, $result) = @_; foreach my $line (split (/\n/, $result)) { if ($line =~ m/^\s*$|^Killed by signal|^Exit |^\d+\.\d+u \d+\./s) { next; } elsif ($line =~ m/^ERROR:/si) { print STDERR "$progname: $type: $line\n"; next; } elsif ($line !~ m@^/.*\t@s) { print STDERR "$progname: $type: ERROR: $line\n"; next; } my @ff = @read_fields; my @vv = split (/\t/, $line); my $file = shift @vv; my %t; while (@ff) { my $k = shift @ff; my $v = shift @vv; $t{$k} = $v; } $file =~ s@^/Users/[^/]+/$music_dir@@si; $file = "CORRUPTED $t{artist} $t{name}" if ($file eq '/missing value'); $all_files{$file} = 1; $table->{$file} = \%t; } }; my %local; my %remote; &$R ("local", \%local, $result1); &$R ("remote", \%remote, $result2); foreach my $file (sort keys (%all_files)) { my $L = $local{$file}; my $R = $remote{$file}; if (!defined($L)) { print STDERR "$file: remote only\n"; } elsif (!defined($R)) { print STDERR "$file: local only\n"; } else { my @fail = (); foreach my $f (@read_fields) { my $LF = $L->{$f}; my $RF = $R->{$f}; my $ok = 0; if ($f eq 'volume adjustment') { # If you write "volume adjustment = 48" and then read it back, # sometimes what got written was actually "49". my $hysteresis = 2; $ok = (($LF >= $RF - $hysteresis && $LF <= $RF + $hysteresis) || ($RF >= $LF - $hysteresis && $RF <= $LF + $hysteresis)); } elsif ($LF =~ m/^-?\d+\.\d+$/s) { # Likewise, the number you write for "start" and "finish" seems # to be randomly permuted at some point after the second decimal. my $hysteresis = 0.002; $ok = (($LF >= $RF - $hysteresis && $LF <= $RF + $hysteresis) || ($RF >= $LF - $hysteresis && $RF <= $LF + $hysteresis)); if (! $ok) { $LF = sprintf("%.4f", $LF); # Truncate for the error msg $RF = sprintf("%.4f", $RF); } } else { $ok = ($LF eq $RF); } if (! $ok) { $LF = "\"$LF\"" if ($LF =~ m/^$|\s/s); $RF = "\"$RF\"" if ($RF =~ m/^$|\s/s); push @fail, "$f: $LF vs $RF"; } } print STDERR "$file: " . join ('; ', @fail) . "\n" if @fail; } } } sub error($) { my ($err) = @_; print STDERR "$progname: $err\n"; exit 1; } sub usage() { print STDERR "usage: $progname [--verbose] [--quiet] [--stars-only]" . " [--diff] [--debug] local-playlist remote-host\n"; exit 1; } sub main() { my ($playlist, $host); my $stars_only = 0; my $force_p = 0; my $diff_p = 0; while ($#ARGV >= 0) { $_ = shift @ARGV; if (m/^--?verbose$/) { $verbose++; } elsif (m/^-v+$/) { $verbose += length($_)-1; } elsif (m/^--?debug$/) { $debug_p++; } elsif (m/^--?quiet$/) { $verbose--; } elsif (m/^--?stars(-only)?$/) { $stars_only++; } elsif (m/^--?force?$/) { $force_p++; } elsif (m/^--?diff?$/) { $diff_p++; } elsif (m/^-./) { usage; } elsif (!$playlist) { $playlist = $_; } elsif (!$host) { $host = $_; } else { usage; } } usage unless ($playlist && $host); if ($diff_p) { diff_files ($host, $playlist); } else { sync_files ($host, $stars_only, $force_p, $playlist); } } main(); exit 0;