#!/usr/bin/perl =pod =head1 NAME tv_imdb - Augment XMLTV listings files with imdb.com data. =head1 SYNOPSIS tv_imdb --imdbdir [--help] [--quiet] [--download] [--movies-only] [--filesort] [--nosystemsort] [--prepStage (1-9,all)] tv_imdb --imdbdir [--help] [--quiet] [--with-keywords] [--with-plot] [--movies-only] [--actors NUMBER] [--stats] [--debug] [--output FILE] [FILE...] tv_imdb --imdbdir --validate-title 'movie title' --validate-year 2004 [--with-keywords] [--with-plot] [--debug] =head1 DESCRIPTION tv_imdb is very similar to tv_cat in semantics (see tv_cat), except whenever a programme appears with "date" entry the title and date are used to look up extra data using XMLTV::IMDB. B<--output FILE> write to FILE rather than standard output. B<--with-keywords> include IMDb keywords in the output file. B<--with-plot> include IMDb plot summary in the output file. B<--actors NUMBER> number of actors from IMDb to add (default=3). B<--quiet> disable all status messages (that normally appear on stderr). B<--download> try to download data files if they are missing (in --prepStage). B<--stats> output grab stats (stats output disabled in --quiet mode). B<--debug> output info from movie matching B<--movies-only> only augment programs that look like movie listings (4 digit E<39>dateE<39> field). All programs are checked against imdb.com data (unless --movies-only is used). For the purposes of tv_imdb, an "exact" match is defined as a case insensitive match against imdb.com data (which may or may not include the transformation of E<39>&E<39> to E<39>andE<39> and vice-versa. If the program includes a 4 digit E<39>dateE<39> field the following matches are attempted, with the first successful match being used: B<1.> an "exact" title/year match against movie titles is done B<2.> an "exact" title match against tv series (and tv mini series) B<3.> an "exact" title match against movie titles with production dates within 2 years of the E<39>dateE<39> value. Unless --movies-only is used, if the program does not include a 4 digit E<39>dateE<39> field the following matches are attempted, the first succeeding match is used: B<1.> an "exact" title match against tv series (and tv mini series) When a match is found in the imdb.com data the following is applied: B<1.> the E<39>titleE<39> field is set to match exactly the title from the imdb.com data. This includes modification of the case to match and any transformations mentioned above. B<2.> if the match is a movie, the E<39>dateE<39> field is set to imdb.com 4 digit year of production. B<3.> the type of match found (Movie, TV Movie, Video Movie, TV Series, or TV Mini Series) is placed in the E<39>categoriesE<39> field. B<4.> the url to the www.imdb.com page is added B<5.> the director is added if the match was a movie or if only one director is listed in the imdb.com data (because some tv series have > 30 directors) B<6.> the top 3 billing actors are added (use -actors [num] to adjust). B<7.> genres added to E<39>categoriesE<39> field (current list of genres are Short, Drama, Comedy, Documentary, Animation, Adult, Action, Family, Romance, Crime, Thriller, Musical, Adventure, Western, Horror, Sci-Fi, Fantasy, Mystery, War, Film-Noir, Music B<8.> imdb user ratings added to E<39>star-ratingsE<39> field. B<9.> imdb keywords added to E<39>keywordE<39> fields (if --with-keywords used). B<10.> imdb plot summary is added (if --with-plot used). =head1 HOWTO In order to use tv_imdb, you need: B<1.> choose a directory location to use for the tv_imdb database (youE<39>ll need about 1 GB of free space), B<2a.> run E<39>tv_imdb --imdbdir --prepStage all --downloadE<39> to download the list files from imdb.com. Or, B<2b> If you have a slow network connection you may prefer to omit the '--download' flag and be prompted for what you need to download by hand. See for the download sites. Then once you have the files rerun without '--download'. Note: '--prepStage' requires up to 520MB of memory. This can be reduced a little by running each prepStage separately, using --prepStage with each of the stages individually (see --help for details). Memory use can be reduced further by using --filesort option when building the database. This will try to use the operating system to sort the interim data files rather than sorting in memory. If this system sort does not work for you then you can use the File::Sort package if it is installed on your system, by also adding the option --nosystemsort (however this method of sorting is very slow). If you specify neither option then Perl will sort the files in memory. If you are only interested in movies, you can reduce the memory required and the size of the database by passing the --movies-only option to the database build, which will exclude tv-series from the database. B<3.> Once you have the database loaded try E<39>cat tv.xml | tv_imdb --imdbdir > tv1.xmlE<39>. Feel free to report any problems with these steps at https://github.com/XMLTV/xmltv/issues. =head1 TESTING The --validate-title and --validate-year flags can be used to validate the information in the tv_imdb database. For exmple: tv_imdb --imdbdir . --validate-title 'Army of Darkness' --validate-year 1994 =head1 BUGS Could use a --configure step just like the grabbers so you do not have to specify the --imdbdir on the command line every time. Also this could step you through the prep stages with more description of what is being done and what is required. Configure could also control the number of actors to add (since some movies have an awful lot), currently we are adding the top 3. How and what to look up needs to be option driven. Needs some more controls for fine tuning "close" matches. For instance, currently it looks like the North America grabber only has date entries for movies, but the imdb.com data contains made for video movies as well as as real movies, so itE<39>s possible to get the wrong data to be inserted. In this case we may want to say "ignore tv series" and "ignore tv mini series". Along with this, weE<39>d want to define what a "close" match is. For instance does a movie by the same title with a date out by 1 year or 2 years considered a match (currently weE<39>re using 2). Nice to haves include: verification/addition of programme MPAA/VCHIP ratings, addition of imdb.com user ratings (by votes) to programmes. Potentially we could expand to include "country of origin", "description", "writer" and "producer" credits, maybe even "commentator". Heh, if the XMLTV.dtd supported it, we could even include urls to head shots of the actors :) =head1 SEE ALSO L =head1 AUTHOR Jerry Veldhuis, jerry@matilda.com =cut use strict; use warnings; use XMLTV; use XMLTV::Version "$XMLTV::VERSION"; use Data::Dumper; use Getopt::Long; use XMLTV::Data::Recursive::Encode; use XMLTV::Usage < [--help] [--quiet] [--download] [--filesort] [--prepStage (1-9,all)] $0 --imdbdir [--help] [--quiet] [--download] [--with-keywords] [--with-plot] [--movies-only] [--actors NUMBER] [--stats] [--debug] [--output FILE] [FILE...] END ; use XMLTV::IMDB; my ($opt_help, $opt_output, $opt_prepStage, $opt_imdbDir, $opt_quiet, $opt_download, $opt_stats, $opt_debug, $opt_movies_only, $opt_with_keywords, $opt_with_plot, $opt_num_actors, $opt_validate_title, $opt_validate_year, $opt_sample, $opt_filesort, $opt_systemsort, ); GetOptions('help' => \$opt_help, 'output=s' => \$opt_output, 'prepStage=s' => \$opt_prepStage, 'imdbdir=s' => \$opt_imdbDir, 'with-keywords' => \$opt_with_keywords, 'with-plot' => \$opt_with_plot, 'movies-only' => \$opt_movies_only, 'actors=s' => \$opt_num_actors, 'quiet' => \$opt_quiet, 'download' => \$opt_download, 'stats' => \$opt_stats, 'debug+' => \$opt_debug, 'validate-title=s' => \$opt_validate_title, 'validate-year=s' => \$opt_validate_year, 'sample=s' => \$opt_sample, 'filesort!' => \$opt_filesort, 'systemsort!' => \$opt_systemsort, ) or usage(0); usage(1) if $opt_help; usage(1) if ( not defined($opt_imdbDir) ); $opt_with_keywords=0 if ( !defined($opt_with_keywords) ); $opt_with_plot=0 if ( !defined($opt_with_plot) ); $opt_num_actors=3 if ( !defined($opt_num_actors) ); $opt_movies_only=0 if ( !defined($opt_movies_only) ); $opt_debug=0 if ( !defined($opt_debug) ); $opt_sample=0 if ( !defined($opt_sample) ); $opt_filesort=0 if ( !defined($opt_filesort) ); $opt_systemsort=1 if ( !defined($opt_systemsort) ); $opt_quiet=(defined($opt_quiet)); if ( !defined($opt_stats) ) { $opt_stats=!$opt_quiet; } else { $opt_stats=(defined($opt_stats)); } $opt_debug=0 if $opt_quiet; if ( defined($opt_prepStage) ) { if ( ! $opt_quiet ) { print STDERR <; # ask for user input chomp($yn); exit(1) if (lc($yn) ne "y"); } } my %options = ('imdbDir' => $opt_imdbDir, 'verbose' => !$opt_quiet, 'showProgressBar' => !$opt_quiet, 'stageToRun' => $opt_prepStage, 'downloadMissingFiles' => $opt_download, 'sample' => $opt_sample, 'filesort' => $opt_filesort, 'systemsort' => $opt_systemsort, 'moviesonly' => $opt_movies_only, ); if ( $opt_prepStage eq "all" ) { my $n=new XMLTV::IMDB::Crunch(%options); if ( !$n ) { exit(1); } for (my $stage=1 ; $stage <= 9 ; $stage++ ) { my $ret=$n->crunchStage($stage); if ( $ret != 0 ) { exit($ret); } } print STDERR "database load complete, let the games begin !\n" if ( !$opt_quiet); exit(0); } else { my $n=new XMLTV::IMDB::Crunch(%options); if ( !$n ) { exit(1); } my $ret=$n->crunchStage(int($opt_prepStage)); if ( $ret == 0 && int($opt_prepStage) == 9 ) { print STDERR "database load complete, let the games begin !\n" if ( !$opt_quiet); } exit($ret); } } elsif ( $opt_download ) { my %options = ('imdbDir' => $opt_imdbDir, 'verbose' => !$opt_quiet, 'showProgressBar' => !$opt_quiet, 'stageToRun' => 'all', 'downloadMissingFiles' => $opt_download, ); my $n=new XMLTV::IMDB::Crunch(%options); if ( !$n ) { exit(1); } exit(0); } my $imdb=new XMLTV::IMDB('imdbDir' => $opt_imdbDir, 'verbose' => $opt_debug, 'cacheLookups' => 1, 'cacheLookupSize' => 1000, 'updateKeywords' => $opt_with_keywords, 'updatePlot' => $opt_with_plot, 'numActors' => $opt_num_actors, ); #$imdb->{verbose}++; if ( my $errline=$imdb->sanityCheckDatabase() ) { print STDERR "$errline"; print STDERR "tv_imdb: you need to use --prepStage to rebuild\n"; exit(1); } if ( !$imdb->openMovieIndex() ) { print STDERR "tv_imdb: open database failed\n"; exit(1); } if ( defined($opt_validate_title) != defined($opt_validate_year) ) { print STDERR "tv_imdb: both --validate-title and --validate-year must be used together\n"; exit(1); } if ( defined($opt_validate_title) && defined($opt_validate_year) ) { my $prog; $prog->{title}->[0]->[0]=$opt_validate_title; $prog->{date}=$opt_validate_year; $imdb->{updateTitles}=0; #print Dumper($prog); my $n=$imdb->augmentProgram($prog, $opt_movies_only); if ( $n ) { $Data::Dumper::Sortkeys = 1; # ensure consistent order of dumped hash #my $encoding; #my $w = new XMLTV::Writer((), encoding => $encoding); #$w->start(shift); #$w->write_programme($n); print Dumper($n); #$w->end(); } $imdb->closeMovieIndex(); exit(0); } # test that movie database works okay my %w_args = (); if (defined $opt_output) { my $fh = new IO::File ">$opt_output"; die "cannot write to $opt_output\n" if not $fh; %w_args = (OUTPUT => $fh); } my $numberOfSeenChannels=0; my $w; my $encoding; # store encoding of input file sub encoding_cb( $ ) { die if defined $w; $encoding = shift; # callback returns the file's encoding $w = new XMLTV::Writer(%w_args, encoding => $encoding); } sub credits_cb( $ ) { $w->start(shift); } my %seen_ch; sub channel_cb( $ ) { my $c = shift; my $id = $c->{id}; $Data::Dumper::Sortkeys = 1; # ensure consistent order of dumped hash if (not defined $seen_ch{$id}) { $w->write_channel($c); $seen_ch{$id} = $c; $numberOfSeenChannels++; } elsif (Dumper($seen_ch{$id}) eq Dumper($c)) { # They're identical, okay. } else { warn "channel $id may differ between two files, " . "picking one arbitrarily\n"; } } sub programme_cb( $ ) { my $prog=shift; # The database made by IMDB.pm is read as iso-8859-1. The xml file may be different (e.g. utf-8). # IMDB::augmentProgram does not re-encode the data it adds, so the output file has invalid characters (bug #440). my $orig_prog = $prog; if (lc($encoding) ne 'iso-8859-1') { # decode the incoming programme $prog = XMLTV::Data::Recursive::Encode->decode($encoding, $prog); } # augmentProgram will now add imdb data as iso-8859-1 my $nprog=$imdb->augmentProgram($prog, $opt_movies_only); if ( $nprog ) { if (lc($encoding) ne 'iso-8859-1') { # re-code the modified programme back to original encoding $nprog = XMLTV::Data::Recursive::Encode->encode($encoding, $nprog); } $prog=$nprog; } else { $prog = $orig_prog; } # we only add movie information to programmes # that have a 'date' element defined (since we need # a year to work with when verifing we got the correct # hit in the imdb data) $w->write_programme($prog); } @ARGV = ('-') if not @ARGV; XMLTV::parsefiles_callback( \&encoding_cb, \&credits_cb, \&channel_cb, \&programme_cb, @ARGV ); # we only get a Writer if the encoding callback gets called if ( $w ) { $w->end(); } if ( $opt_stats ) { print STDERR $imdb->getStatsLines($numberOfSeenChannels); } $imdb->closeMovieIndex(); exit(0);