package tests::DlfStoreTest; use strict; use base qw/ Lire::Test::TestCase tests::TestStoreFixture tests::ChartTypesFixture tests::OutputFormatsFixture/; use Lire::DlfStore; use Lire::Utils qw/create_file tempdir/; use Lire::Report; use Lire::ReportJob; use Lire::ReportSchedule; use Lire::DlfSchema; use Lire::DlfConverter; use Lire::DlfAnalyser; use tests::helpers::TestDerivedAnalyzer; use tests::helpers::TestExtendedAnalyzer; use Lire::PluginManager; use Lire::Test::Mock; use Lire::ReportConfig; use Lire::ReportSection; use Lire::ReportSpec; use Lire::Config::Index; use Lire::Config::TypeSpec; use Lire::Config::Parser; use File::Path qw/mkpath/; use File::Copy; use Time::Local; #our @TESTS = qw//; sub new { my $self = shift()->SUPER::new( @_ ); $self->init(); $self->{'workingdir'} = tempdir( __PACKAGE__ . "XXXXXX", 'CLEANUP' => 1, TMPDIR => 1, ); my $spec = new Lire::Config::ConfigSpec(); my $spec_string = new Lire::Config::StringSpec( 'name' => 'String' ); $spec->add( $spec_string ); $spec->add( new Lire::Config::ListSpec( 'name' => 'reports' ), ); $spec->get( 'reports' )->add( new Lire::Config::ReportSpec( 'name' => 'report' ) ); $spec->add( new Lire::Config::ListSpec( 'name' => 'report_jobs' ) ); $self->{'spec'} = $spec; return $self; } sub set_up { my $self = $_[0]; $self->SUPER::set_up(); $self->set_up_test_schema(); $self->{'cfg'}{'lr_week_numbering'} = 'ISO'; $self->{'cfg'}{'_lr_config_spec'} = $self->{'spec'}; $self->set_up_tz( 'EST' ); $self->{'_old_index_reg'} = \%Lire::Config::Index::REGISTRY; %Lire::Config::Index::REGISTRY = (); return; } sub tear_down { my $self = $_[0]; Lire::Test::Mock->reset_factories(); *Lire::Config::Index::REGISTRY = $self->{'_old_index_reg'}; # Since the DlfStreamSpec are built at runtime, clear # it so that the list of extension schemas is alway computed anew my $spec = $self->lire_default_config_spec()->get( 'streams_config' ); $spec->{'components'} = []; $spec->{'index'} = {}; $self->tear_down_test_store() if $self->{'store'}; $self->SUPER::tear_down(); return; } sub test_open_create { my $self = $_[0]; my $path = $self->{'workingdir'} . "/test_open_create"; my $store = Lire::DlfStore->open( $path, 1 ); $self->assert_not_null( $store, "Lire::DlfStore->open returned undef" ); $self->assert_equals( $path, $store->path() ); $self->assert( -d $path, "open() didn't create the directory" ); $self->assert_not_null( $store->{'_dbh'}, "_dbh attribute is undef" ); $self->assert( -f "$path/dlf.db", "DLF database '$path/dlf.db' wasn't created" ); $self->assert( -f "$path/lock", "open() didn't create the lock file" ); $self->_check_dbh( $store ); $self->assert_isa( 'Lire::Config::ConfigFile', $store->{'_config'} ); $self->assert_str_equals( "$path/config.xml", $store->{'_config'}->filename()); local $SIG{'__WARN__'} = sub { $self->annotate( @_ ) }; $self->assert_deep_equals( { 'String' => '', 'reports' => [], 'report_jobs' => [], }, $store->{'_config'}->as_value() ); $store->close(); } sub test_open { my $self = $_[0]; my $registered_functions = 0; my $registered_aggregates = 0; no warnings 'redefine'; local *Lire::SQLExt::Registry::register_functions = sub { $registered_functions = 1; }; local *Lire::SQLExt::Registry::register_aggregates = sub { $registered_aggregates = 1; }; my $bad_path = $self->{'workingdir'} . "/bad_store"; mkdir $bad_path; $self->assert_dies( qr/Invalid DlfStore: \'$bad_path\'/, sub { Lire::DlfStore->open( $bad_path ) } ); my $path = $self->{'workingdir'} . "/test_open"; $self->assert_dies( qr/DlfStore \'$path\' doesn't exist/, sub { Lire::DlfStore->open( $path ) } ); my $store = Lire::DlfStore->open( $path, 1 ); $self->assert_not_null( $store, "open() returned undef" ); $store->{'_config'}->get( 'String' )->set( 'wawa string' ); $store->close(); $store = Lire::DlfStore->open( $path ); $self->assert_str_equals( 'wawa string', $store->{'_config'}->get( 'String' )->get() ); $self->assert_not_null( $store, "open() returned undef" ); $self->assert_equals( $path, $store->path ); $self->_check_dbh( $store ); $self->assert( Lire::Config::Index->has_index( 'store_report_configurations' ) ? 1 : 0, 'has_index()' ); $self->assert_isa( 'Lire::Config::ReportConfigIndex', Lire::Config::Index->get_index( 'store_report_configurations' ) ); $store->close(); $self->assert( $registered_functions, 'open() should register SQLExt functions' ); $self->assert( $registered_aggregates, 'open() should register SQLExt aggregates' ); } sub _check_dbh { my ($self, $store ) = @_; $self->assert( $store->{'_dbh'}, "_dbh attribute is undef" ); $self->assert_equals( "SQLite2", $store->{'_dbh'}{'Driver'}{'Name'} ); $self->assert( $store->{'_dbh'}{'RaiseError'}, "RaiseError isn't enabled on _dbh" ); $self->assert( !$store->{'_dbh'}{'AutoCommit'}, "AutoCommit isn't turn off on _dbh" ); } sub test_close { my $self = $_[0]; my $path = $self->{'workingdir'} . "/test_close"; my $store = Lire::DlfStore->open( $path, 1); $self->assert_not_null( $store, "open()) returned undef" ); my $warnings = ""; local $SIG{'__WARN__'} = sub { $warnings .= join "", @_ }; $self->assert( ! -e "$path/config.xml" ); $store->close(); $self->annotate( $warnings ); $self->assert( ! -f "$path/lock", "close()) didn't remove the lock file" ); $self->assert_null( $store->{'_dbh'}, "_dbh wasn't closed?" ); $self->assert( !$warnings, "warnings were generated during close()" ); $self->assert( -f "$path/config.xml", 'Expected config.xml to be created' ); $self->assert( ! Lire::Config::Index->has_index( 'store_report_configurations' ) ? 1 : 0, "!has_index()" ); } sub test_locked_open { my $self = $_[0]; my $path = $self->{'workingdir'} . "/test_locked_open"; my $store = Lire::DlfStore->open( $path, 1); $store->close(); create_file( "$path/lock", getppid . "\n" ); $self->annotate( <assert_dies( qr/DlfStore '$path' is locked by process/, sub { Lire::DlfStore->open( $path ) } ); } sub test_stale_lock { my $self = $_[0]; my $path = $self->{'workingdir'} . "/test_state_lock"; my $store = Lire::DlfStore->open( $path, 1); $self->assert_not_null( $store, "open()) returned undef" ); $store->close; open LOCK, "> $path/lock" or die "can't create bogus lock file\n"; print LOCK "-2\n"; close LOCK; $store = Lire::DlfStore->open( $path ); $self->assert_not_null( $store, "open() returned undef" ); $store->close; } sub test_open_dlf_stream { my $self = $_[0]; my $path = $self->{'workingdir'} . "/test_open_stream"; my $store = Lire::DlfStore->open( $path, 1 ); $self->assert_not_null( $store, "open() returned undef" ); my @streams = $store->dlf_streams; $self->assert_equals( 0, scalar @streams ); $self->assert( ! $store->has_dlf_stream( "test" ), "store shouldn't have a test DLF stream." ); $self->assert_dies( qr/no DLF stream 'test' in this store/, sub { $store->open_dlf_stream( "test", "r" ) } ); my $s = $store->open_dlf_stream( "test", "w" ); $self->assert_isa( "Lire::DlfStream", $s ); my $stream_r = $store->open_dlf_stream( "test", "r" ); $self->assert_isa( "Lire::DlfStream", $stream_r ); $self->assert_equals( 'r', $stream_r->mode() ); my $stream_w2 = $store->open_dlf_stream( "test", "w" ); $self->assert_isa( "Lire::DlfStream", $stream_w2 ); $self->assert_equals( 'w', $stream_w2->mode() ); $s->close(); $stream_r->close(); $stream_w2->close(); $s = $store->open_dlf_stream( "test", "r" ); $self->assert_isa( "Lire::DlfStream", $s ); $self->assert( $store->has_dlf_stream( "test"), "has_dlf_stream() should have succeeded" ); @streams = $store->dlf_streams(); $self->assert_deep_equals( [ "test"], \@streams ); $s->close(); } sub test_report_filename { my $self = $_[0]; my $path = "$self->{'workingdir'}/test_report_filename"; my $store = bless { '_store_path' => $path }, 'Lire::DlfStore'; $self->assert_dies( qr/'period' parameter should be one of 'hourly', 'daily', 'weekly', 'monthly' or 'yearly'/, sub { $store->_report_filename( 'test', 'unique', time ) } ); my $jan7_2004 = timelocal( 0, 0, 0, 7, 0, 2004 ); $self->assert_str_equals( "$path/hourly_reports/test/200401/07/00.xml", $store->_report_filename( 'test', 'hourly', $jan7_2004 ) ); $self->assert_str_equals( "$path/daily_reports/test/200401/07.xml", $store->_report_filename( 'test', 'daily', $jan7_2004 ) ); $self->assert_str_equals( "$path/weekly_reports/test/2004/02.xml", $store->_report_filename( 'test', 'weekly', $jan7_2004 ) ); $self->assert_str_equals( "$path/monthly_reports/test/2004/01.xml", $store->_report_filename( 'test', 'monthly', $jan7_2004 ) ); $self->assert_str_equals( "$path/yearly_reports/test/2004.xml", $store->_report_filename( 'test', 'yearly', $jan7_2004 ) ); } sub test_put_report { my $self = $_[0]; my $path = "$self->{'workingdir'}/test_put_report"; my $store = Lire::DlfStore->open( $path, 1 ); my $feb14_2004 = timelocal( 0, 5, 12, 14, 1, 2004 ); my $report = new Lire::Report( 'hourly', $feb14_2004, $feb14_2004 + 3600 ); my $job = new Lire::ReportJob( 'aTest', 'test' ); my $sched = new Lire::ReportSchedule( 'hourly', new Lire::ReportConfig() ); my $file = $store->put_report( $job, $sched, $report ); $self->assert( -s $file, "report doesn't exists in store" ); $self->assert_str_equals( "$path/hourly_reports/aTest/200402/14/12.xml", $file ); } sub set_up_report_cfg { my $self = $_[0]; $self->{'cfg'}{'lr_reports_path'} = [ "$self->{'testdir'}/reports" ]; $self->{'test_cfg'} = new Lire::ReportConfig(); my $section = new Lire::ReportSection( 'test', 'Title' ); my $spec = Lire::ReportSpec->load( 'test', 'top-files' ); $spec->subreport_id( 'my_id' ); $section->add_report( $spec ); $spec = Lire::ReportSpec->load( 'test', 'sessions-by-user_class' ); $spec->subreport_id( 'sessions-by-user_class.0' ); $section->add_report( $spec ); $self->{'test_cfg'}->add_section( $section ); } sub test_find_report_source_dlf_only { my $self = $_[0]; my $jan25_2003_noon = timelocal( 0, 5, 12, 25, 0, 2003 ); $self->set_up_test_store(); $self->set_up_report_cfg(); my $store = $self->{'store'}; my $job = new Lire::ReportJob( 'aJob' ); my $sched = new Lire::ReportSchedule( 'hourly', $self->{'test_cfg'} ); $self->assert_deep_equals( { 'source' => 'dlf', 'start' => 1043514000, 'end' => 1043514000 + 3600, 'coverage' => 100 }, $store->find_report_source( $job, $sched, $jan25_2003_noon, ) ); my $jan25_2003_1600 = timelocal( 0, 5, 16, 25, 0, 2003 ); $sched = new Lire::ReportSchedule( 'hourly', $self->{'test_cfg'} ); $self->assert_deep_equals( { 'source' => 'none', 'coverage' => 0, }, $store->find_report_source( $job, $sched, $jan25_2003_1600 )); $sched = new Lire::ReportSchedule( 'daily', $self->{'test_cfg'} ); $self->assert_deep_equals( { 'source' => 'dlf', 'start' => 1043514000, 'end' => 1043526410, 'coverage' => 14, }, $store->find_report_source( $job, $sched, $jan25_2003_noon, ) ); } sub test_find_report_source { my $self = $_[0]; my $jan25_2003_noon = timelocal( 0, 5, 12, 25, 0, 2003 ); $self->set_up_test_store(); $self->set_up_report_cfg(); my $store = $self->{'store'}; my $job = new Lire::ReportJob( 'aJob' ); my $sched = new Lire::ReportSchedule( 'weekly', $self->{'test_cfg'} ); $self->assert_deep_equals( { 'source' => 'dlf', 'start' => 1043514000, 'end' => 1043526410, 'coverage' => 2, }, $store->find_report_source( $job, $sched, $jan25_2003_noon, ) ); my $dir = "$self->{'store'}{'_store_path'}/daily_reports/aJob/200301"; mkpath( $dir, 0, 0755); for my $day ( 23, 24, 26, 27 ) { create_file( "$dir/$day.xml", '' ); } $self->assert_deep_equals( { 'source' => 'merging', 'start' => 1043298000, 'end' => 1043643600, 'coverage' => 57, 'reports' => [ "$dir/23.xml", "$dir/24.xml", "$dir/26.xml" ], 'days' => [ '2003-01-23', '2003-01-24', '2003-01-26' ], }, $store->find_report_source( $job, $sched, $jan25_2003_noon, ) ); for my $day ( 19, 20, 21, 22, 25 ) { create_file( "$dir/$day.xml", '' ); } $self->assert_deep_equals( { 'source' => 'merging', 'start' => 1043038800, 'end' => 1043643600, 'coverage' => 100, 'reports' => [ "$dir/20.xml", "$dir/21.xml", "$dir/22.xml", "$dir/23.xml", "$dir/24.xml", "$dir/25.xml", "$dir/26.xml" ], 'days' => [ '2003-01-20', '2003-01-21', '2003-01-22', '2003-01-23', '2003-01-24', '2003-01-25', '2003-01-26', ], }, $store->find_report_source( $job, $sched, $jan25_2003_noon, ) ); } sub set_up_plugins { my $self = $_[0]; $self->set_up_plugin_mgr(); my $mgr = Lire::PluginManager->instance(); $mgr->register_plugin( new tests::helpers::TestDerivedAnalyzer() ); $mgr->register_plugin( new tests::helpers::TestExtendedAnalyzer() ); my $converter = new_proxy Lire::Test::Mock( 'Lire::DlfConverter' ); $converter->set_result( 'name' => 'mydlf', 'title' => 'My DLF', 'schemas' => sub { return ( 'ftp', 'www' ) } ); $mgr->register_plugin( $converter ); my $analyser = new_proxy Lire::Test::Mock( 'Lire::DlfAnalyser' ); $analyser->set_result( 'name' => 'myanalyser', 'title' => 'My DLF', 'description' => 'My description', 'src_schema' => 'test-derived', 'dst_schema' => 'test-another' ); $mgr->register_plugin( $analyser ); } sub set_up_store_config { my $self = $_[0]; my $config = $self->{'store'}->config(); my $jobspec = $self->{'store'}->config()->spec()->get( 'import_jobs' )->get( 'import_job' ); my $job = $jobspec->instance(); $job->set( $jobspec->get( 'name' )->instance( 'value' => 'myjob' ) ); $job->set( $jobspec->get( 'period' )->instance( 'value' => 'hourly' ) ); $job->set( $jobspec->get( 'service' )->instance( 'value' => 'mydlf' ) ); $job->set( $jobspec->get( 'log_file' )->instance( 'value' => 'test.log' )); $config->get( 'import_jobs' )->append( $job ); } sub test_get_stream_config { my $self = $_[0]; $self->set_up_test_store(); my $store = $self->{'store'}; $self->assert_dies( qr/schema \'wawa\' doesn\'t exist/, sub { $store->get_stream_config( 'wawa' ) } ); $self->assert_num_equals( 0, scalar $store->{'_config'}->get( 'streams_config' )->elements() ); my $cfg = $store->get_stream_config( 'test-extended' ); $self->assert_isa( 'Lire::Config::Dictionary', $cfg ); $self->assert_str_equals( 'test-extended', $cfg->name() ); $self->assert_isa( 'Lire::Config::DlfStreamSpec', $cfg->spec() ); $self->assert_deep_equals( [ $cfg ], [ $store->{'_config'}->get( 'streams_config' )->elements() ] ); $self->assert_str_equals( $cfg, $store->get_stream_config( 'test-extended' ) ); } sub test_dlf_streams { my $self = $_[0]; my $store = Lire::DlfStore->open( "$self->{'workingdir'}/test_dlf_streams", 1 ); $store->open_dlf_stream( 'test', 'w' )->close(); $store->open_dlf_stream( 'test-extended', 'w' )->close(); $store->open_dlf_stream( 'test-derived', 'w' )->close(); $self->assert_deep_equals( [ 'test', 'test-derived', 'test-extended' ], [ sort $store->dlf_streams() ] ); $store->close(); } sub test_configured_dlf_streams { my $self = $_[0]; local %Lire::DlfSchema::SCHEMA_CACHE = ( 'www' => 1, 'test-another' => 1 ); $self->set_up_plugins(); $self->set_up_test_store(); $self->set_up_store_config(); $self->assert_deep_equals( [ 'test', 'test-another', 'test-derived', 'test-extended', 'www' ], [ $self->{'store'}->configured_dlf_streams()] ); my $test_cfg = $self->{'store'}->get_stream_config( 'test' ); $test_cfg->get( 'test-derived' )->set_plugin( 'none' ); $self->assert_deep_equals( [ 'test', 'test-extended', 'www' ], [ $self->{'store'}->configured_dlf_streams()] ); } sub set_up_test_clean_streams { my $self = $_[0]; $self->{'cfg'}{'_lr_config_spec'} = $self->lire_default_config_spec(); my $now = time; my $one_week_ago = $now - (7 *86400) - 1; my $yesterday = $now - 86400 - 1; $self->{'astore'} = Lire::DlfStore->open( "$self->{'workingdir'}/test_clean_streams", 1 ); my $test = $self->{'astore'}->open_dlf_stream( 'test', 'w' ); $test->write_dlf( { 'time_start' => $one_week_ago } ); $test->write_dlf( { 'time_start' => $yesterday } ); $test->write_dlf( { 'time_start' => $now } ); $test->close(); my $derived = $self->{'astore'}->open_dlf_stream( 'test-derived', 'w' ); $derived->write_dlf( { 'session_start' => $one_week_ago } ); $derived->write_dlf( { 'session_start' => $yesterday } ); $derived->write_dlf( { 'session_start' => $now } ); $derived->close(); $self->{'astore'}->get_stream_config( 'test' )->get( 'keep_days' )->set( 1 ); $self->{'astore'}->get_stream_config( 'test-derived' )->get( 'keep_days' )->set( 0 ); } sub test_clean_streams { my $self = $_[0]; $self->set_up_test_clean_streams(); my $test = $self->{'astore'}->open_dlf_stream( 'test', 'w' ); my $derived = $self->{'astore'}->open_dlf_stream( 'test-derived', 'w' ); $self->assert_num_equals( 3, $test->nrecords() ); $self->assert_num_equals( 3, $derived->nrecords() ); $self->{'astore'}->clean_streams(); $self->assert_num_equals( 1, $test->nrecords() ); $self->assert_num_equals( 3, $derived->nrecords() ); } sub set_up_test_run_analysers { my $self = $_[0]; $self->set_up_plugin_mgr(); $self->set_up_test_store(); my $mgr = Lire::PluginManager->instance(); $mgr->register_plugin( new tests::helpers::TestDerivedAnalyzer() ); $mgr->register_plugin( new tests::helpers::TestExtendedAnalyzer() ); $self->{'store'}->get_stream_config( 'test' )->get( 'test-extended' )->set_plugin( 'none' ); Lire::Test::Mock->set_mock_factory( 'Lire::DlfAnalyserProcess' ); $self->{'store'} = new_proxy Lire::Test::Mock( $self->{'store'} ); } sub test_run_analysers { my $self = $_[0]; $self->set_up_test_run_analysers(); $self->{'store'}->run_analysers( 'test', 'my_source' ); my $analysers = Lire::Test::Mock->mock_instances( 'Lire::DlfAnalyserProcess' ); $self->assert_num_equals( 1, scalar @$analysers ); $self->assert_str_equals( 'my_source', $analysers->[0]->dlf_source() ); $self->assert_str_equals( $self->{'store'}, $analysers->[0]->dlf_store() ); $self->assert_str_equals( 'derived', $analysers->[0]->dlf_analyser() ); $self->assert_deep_equals( {}, $analysers->[0]->dlf_analyser_config() ); $self->assert_num_equals( 1, $analysers->[0]->invocation_count( 'run_analysis_job' ) ); $self->assert_num_equals( 2, $self->{'store'}->invocation_count( 'run_analysers' ) ); $self->assert_deep_equals( [ $self->{'store'}, 'test-derived', $analysers->[0]->job_id() ], $self->{'store'}->get_invocation( 'run_analysers', 1 ) ); } sub set_up_migrate_report_jobs { my $self = $_[0]; $self->set_up_plugin_mgr(); my $mgr = Lire::PluginManager->instance(); $mgr->register_plugin( new tests::helpers::TestDerivedAnalyzer() ); $mgr->register_plugin( new tests::helpers::TestExtendedAnalyzer() ); my $analyser = new_proxy Lire::Test::Mock( 'Lire::DlfAnalyser' ); $analyser->set_result( 'name' => 'another', 'title' => 'Another Catogoriser', 'description' => 'A fake Categoriser', 'src_schema' => 'test-derived', 'dst_schema' => 'test-extended', 'analyse' => '' ); Lire::PluginManager->register_plugin( $analyser ); my $converter = new_proxy Lire::Test::Mock( 'Lire::DlfConverter' ); $converter->set_result( 'name' => 'test_newapi', 'title' => 'Fake DlfConverter', 'description' => 'A fake DlfConverter', 'schemas' => 'test' ); Lire::PluginManager->register_plugin( $converter ); $self->{'cfg'}{'lr_reports_path'} = [ "$self->{'testdir'}/reports" ]; $self->{'cfg'}{'lr_filters_path'} = [ "$self->{'testdir'}/filters" ]; $self->set_up_test_store( 0 ); copy( "$self->{'testdir'}/data/test.cfg", "$self->{'tmpdir'}" ); open my $ofh, "> $self->{'store_path'}/config.xml" or $self->error( "open >: $!" ); open my $ifh, "$self->{'testdir'}/data/jobs-config-15.xml" or $self->error( "open <: $!" ); while ( my $line = <$ifh> ) { $line =~ s!test.cfg!$self->{'tmpdir'}\/test.cfg!; print $ofh $line; } close $ofh; close $ifh; $self->set_up_chart_types( 0 ); $self->set_up_output_formats( 0 ); return; } sub test_migrate_report_jobs { my $self = $_[0]; $self->set_up_migrate_report_jobs(); my $store = Lire::DlfStore->open( $self->{'store_path'} ); $store->close(); my $parser = new Lire::Config::Parser( 'spec' => $self->{'cfg'}{'_lr_config_spec'} ); my $expected = $parser->load_config_file( "$self->{'testdir'}/data/jobs-config-15-migrated.xml" ); my $migrated = $parser->load_config_file( "$self->{'store_path'}/config.xml" ); $expected->filename( $migrated->filename() ); $self->assert_deep_equals( $expected, $migrated ); } 1;