package tests::DlfStoreTest;
use strict;
use base qw/ Lire::Test::TestCase tests::TestStoreFixture
tests::ChartTypesFixture tests::OutputFormatsFixture/;
use Lire::DlfStore;
use Lire::Utils qw/create_file tempdir/;
use Lire::Report;
use Lire::ReportJob;
use Lire::ReportSchedule;
use Lire::DlfSchema;
use Lire::DlfConverter;
use Lire::DlfAnalyser;
use tests::helpers::TestDerivedAnalyzer;
use tests::helpers::TestExtendedAnalyzer;
use Lire::PluginManager;
use Lire::Test::Mock;
use Lire::ReportConfig;
use Lire::ReportSection;
use Lire::ReportSpec;
use Lire::Config::Index;
use Lire::Config::TypeSpec;
use Lire::Config::Parser;
use File::Path qw/mkpath/;
use File::Copy;
use Time::Local;
#our @TESTS = qw//;
sub new {
my $self = shift()->SUPER::new( @_ );
$self->init();
$self->{'workingdir'} = tempdir( __PACKAGE__ . "XXXXXX",
'CLEANUP' => 1, TMPDIR => 1,
);
my $spec = new Lire::Config::ConfigSpec();
my $spec_string = new Lire::Config::StringSpec( 'name' => 'String' );
$spec->add( $spec_string );
$spec->add( new Lire::Config::ListSpec( 'name' => 'reports' ), );
$spec->get( 'reports' )->add( new Lire::Config::ReportSpec( 'name' => 'report' ) );
$spec->add( new Lire::Config::ListSpec( 'name' => 'report_jobs' ) );
$self->{'spec'} = $spec;
return $self;
}
sub set_up {
my $self = $_[0];
$self->SUPER::set_up();
$self->set_up_test_schema();
$self->{'cfg'}{'lr_week_numbering'} = 'ISO';
$self->{'cfg'}{'_lr_config_spec'} = $self->{'spec'};
$self->set_up_tz( 'EST' );
$self->{'_old_index_reg'} = \%Lire::Config::Index::REGISTRY;
%Lire::Config::Index::REGISTRY = ();
return;
}
sub tear_down {
my $self = $_[0];
Lire::Test::Mock->reset_factories();
*Lire::Config::Index::REGISTRY = $self->{'_old_index_reg'};
# Since the DlfStreamSpec are built at runtime, clear
# it so that the list of extension schemas is alway computed anew
my $spec = $self->lire_default_config_spec()->get( 'streams_config' );
$spec->{'components'} = [];
$spec->{'index'} = {};
$self->tear_down_test_store()
if $self->{'store'};
$self->SUPER::tear_down();
return;
}
sub test_open_create {
my $self = $_[0];
my $path = $self->{'workingdir'} . "/test_open_create";
my $store = Lire::DlfStore->open( $path, 1 );
$self->assert_not_null( $store, "Lire::DlfStore->open returned undef" );
$self->assert_equals( $path, $store->path() );
$self->assert( -d $path, "open() didn't create the directory" );
$self->assert_not_null( $store->{'_dbh'}, "_dbh attribute is undef" );
$self->assert( -f "$path/dlf.db",
"DLF database '$path/dlf.db' wasn't created" );
$self->assert( -f "$path/lock", "open() didn't create the lock file" );
$self->_check_dbh( $store );
$self->assert_isa( 'Lire::Config::ConfigFile', $store->{'_config'} );
$self->assert_str_equals( "$path/config.xml",
$store->{'_config'}->filename());
local $SIG{'__WARN__'} = sub { $self->annotate( @_ ) };
$self->assert_deep_equals( { 'String' => '',
'reports' => [],
'report_jobs' => [],
},
$store->{'_config'}->as_value() );
$store->close();
}
sub test_open {
my $self = $_[0];
my $registered_functions = 0;
my $registered_aggregates = 0;
no warnings 'redefine';
local *Lire::SQLExt::Registry::register_functions =
sub {
$registered_functions = 1;
};
local *Lire::SQLExt::Registry::register_aggregates =
sub {
$registered_aggregates = 1;
};
my $bad_path = $self->{'workingdir'} . "/bad_store";
mkdir $bad_path;
$self->assert_dies( qr/Invalid DlfStore: \'$bad_path\'/,
sub { Lire::DlfStore->open( $bad_path ) } );
my $path = $self->{'workingdir'} . "/test_open";
$self->assert_dies( qr/DlfStore \'$path\' doesn't exist/,
sub { Lire::DlfStore->open( $path ) } );
my $store = Lire::DlfStore->open( $path, 1 );
$self->assert_not_null( $store, "open() returned undef" );
$store->{'_config'}->get( 'String' )->set( 'wawa string' );
$store->close();
$store = Lire::DlfStore->open( $path );
$self->assert_str_equals( 'wawa string',
$store->{'_config'}->get( 'String' )->get() );
$self->assert_not_null( $store, "open() returned undef" );
$self->assert_equals( $path, $store->path );
$self->_check_dbh( $store );
$self->assert( Lire::Config::Index->has_index( 'store_report_configurations' ) ? 1 : 0,
'has_index()' );
$self->assert_isa( 'Lire::Config::ReportConfigIndex',
Lire::Config::Index->get_index( 'store_report_configurations' ) );
$store->close();
$self->assert( $registered_functions,
'open() should register SQLExt functions' );
$self->assert( $registered_aggregates,
'open() should register SQLExt aggregates' );
}
sub _check_dbh {
my ($self, $store ) = @_;
$self->assert( $store->{'_dbh'}, "_dbh attribute is undef" );
$self->assert_equals( "SQLite2", $store->{'_dbh'}{'Driver'}{'Name'} );
$self->assert( $store->{'_dbh'}{'RaiseError'},
"RaiseError isn't enabled on _dbh" );
$self->assert( !$store->{'_dbh'}{'AutoCommit'},
"AutoCommit isn't turn off on _dbh" );
}
sub test_close {
my $self = $_[0];
my $path = $self->{'workingdir'} . "/test_close";
my $store = Lire::DlfStore->open( $path, 1);
$self->assert_not_null( $store, "open()) returned undef" );
my $warnings = "";
local $SIG{'__WARN__'} = sub { $warnings .= join "", @_ };
$self->assert( ! -e "$path/config.xml" );
$store->close();
$self->annotate( $warnings );
$self->assert( ! -f "$path/lock", "close()) didn't remove the lock file" );
$self->assert_null( $store->{'_dbh'}, "_dbh wasn't closed?" );
$self->assert( !$warnings, "warnings were generated during close()" );
$self->assert( -f "$path/config.xml",
'Expected config.xml to be created' );
$self->assert( ! Lire::Config::Index->has_index( 'store_report_configurations' ) ? 1 : 0, "!has_index()" );
}
sub test_locked_open {
my $self = $_[0];
my $path = $self->{'workingdir'} . "/test_locked_open";
my $store = Lire::DlfStore->open( $path, 1);
$store->close();
create_file( "$path/lock", getppid . "\n" );
$self->annotate( <<EORANT );
assert_dies() is hosed because \$@ is lost in this particular case, ask
the "did-too-much-acid-in-their-time" perl's authors for a plausible
reason.
EORANT
$self->assert_dies( qr/DlfStore '$path' is locked by process/,
sub { Lire::DlfStore->open( $path ) } );
}
sub test_stale_lock {
my $self = $_[0];
my $path = $self->{'workingdir'} . "/test_state_lock";
my $store = Lire::DlfStore->open( $path, 1);
$self->assert_not_null( $store, "open()) returned undef" );
$store->close;
open LOCK, "> $path/lock"
or die "can't create bogus lock file\n";
print LOCK "-2\n";
close LOCK;
$store = Lire::DlfStore->open( $path );
$self->assert_not_null( $store, "open() returned undef" );
$store->close;
}
sub test_open_dlf_stream {
my $self = $_[0];
my $path = $self->{'workingdir'} . "/test_open_stream";
my $store = Lire::DlfStore->open( $path, 1 );
$self->assert_not_null( $store, "open() returned undef" );
my @streams = $store->dlf_streams;
$self->assert_equals( 0, scalar @streams );
$self->assert( ! $store->has_dlf_stream( "test" ),
"store shouldn't have a test DLF stream." );
$self->assert_dies( qr/no DLF stream 'test' in this store/,
sub { $store->open_dlf_stream( "test", "r" ) } );
my $s = $store->open_dlf_stream( "test", "w" );
$self->assert_isa( "Lire::DlfStream", $s );
my $stream_r = $store->open_dlf_stream( "test", "r" );
$self->assert_isa( "Lire::DlfStream", $stream_r );
$self->assert_equals( 'r', $stream_r->mode() );
my $stream_w2 = $store->open_dlf_stream( "test", "w" );
$self->assert_isa( "Lire::DlfStream", $stream_w2 );
$self->assert_equals( 'w', $stream_w2->mode() );
$s->close();
$stream_r->close();
$stream_w2->close();
$s = $store->open_dlf_stream( "test", "r" );
$self->assert_isa( "Lire::DlfStream", $s );
$self->assert( $store->has_dlf_stream( "test"),
"has_dlf_stream() should have succeeded" );
@streams = $store->dlf_streams();
$self->assert_deep_equals( [ "test"], \@streams );
$s->close();
}
sub test_report_filename {
my $self = $_[0];
my $path = "$self->{'workingdir'}/test_report_filename";
my $store = bless { '_store_path' => $path }, 'Lire::DlfStore';
$self->assert_dies( qr/'period' parameter should be one of 'hourly', 'daily', 'weekly', 'monthly' or 'yearly'/,
sub { $store->_report_filename( 'test', 'unique',
time ) } );
my $jan7_2004 = timelocal( 0, 0, 0, 7, 0, 2004 );
$self->assert_str_equals( "$path/hourly_reports/test/200401/07/00.xml",
$store->_report_filename( 'test', 'hourly',
$jan7_2004 ) );
$self->assert_str_equals( "$path/daily_reports/test/200401/07.xml",
$store->_report_filename( 'test', 'daily',
$jan7_2004 ) );
$self->assert_str_equals( "$path/weekly_reports/test/2004/02.xml",
$store->_report_filename( 'test', 'weekly',
$jan7_2004 ) );
$self->assert_str_equals( "$path/monthly_reports/test/2004/01.xml",
$store->_report_filename( 'test', 'monthly',
$jan7_2004 ) );
$self->assert_str_equals( "$path/yearly_reports/test/2004.xml",
$store->_report_filename( 'test', 'yearly',
$jan7_2004 ) );
}
sub test_put_report {
my $self = $_[0];
my $path = "$self->{'workingdir'}/test_put_report";
my $store = Lire::DlfStore->open( $path, 1 );
my $feb14_2004 = timelocal( 0, 5, 12, 14, 1, 2004 );
my $report = new Lire::Report( 'hourly', $feb14_2004, $feb14_2004 + 3600 );
my $job = new Lire::ReportJob( 'aTest', 'test' );
my $sched = new Lire::ReportSchedule( 'hourly', new Lire::ReportConfig() );
my $file = $store->put_report( $job, $sched, $report );
$self->assert( -s $file, "report doesn't exists in store" );
$self->assert_str_equals( "$path/hourly_reports/aTest/200402/14/12.xml",
$file );
}
sub set_up_report_cfg {
my $self = $_[0];
$self->{'cfg'}{'lr_reports_path'} = [ "$self->{'testdir'}/reports" ];
$self->{'test_cfg'} = new Lire::ReportConfig();
my $section = new Lire::ReportSection( 'test', 'Title' );
my $spec = Lire::ReportSpec->load( 'test', 'top-files' );
$spec->subreport_id( 'my_id' );
$section->add_report( $spec );
$spec = Lire::ReportSpec->load( 'test', 'sessions-by-user_class' );
$spec->subreport_id( 'sessions-by-user_class.0' );
$section->add_report( $spec );
$self->{'test_cfg'}->add_section( $section );
}
sub test_find_report_source_dlf_only {
my $self = $_[0];
my $jan25_2003_noon = timelocal( 0, 5, 12, 25, 0, 2003 );
$self->set_up_test_store();
$self->set_up_report_cfg();
my $store = $self->{'store'};
my $job = new Lire::ReportJob( 'aJob' );
my $sched = new Lire::ReportSchedule( 'hourly', $self->{'test_cfg'} );
$self->assert_deep_equals( { 'source' => 'dlf',
'start' => 1043514000,
'end' => 1043514000 + 3600,
'coverage' => 100
}, $store->find_report_source( $job, $sched,
$jan25_2003_noon,
) );
my $jan25_2003_1600 = timelocal( 0, 5, 16, 25, 0, 2003 );
$sched = new Lire::ReportSchedule( 'hourly', $self->{'test_cfg'} );
$self->assert_deep_equals( { 'source' => 'none',
'coverage' => 0,
},
$store->find_report_source( $job, $sched,
$jan25_2003_1600 ));
$sched = new Lire::ReportSchedule( 'daily', $self->{'test_cfg'} );
$self->assert_deep_equals( { 'source' => 'dlf',
'start' => 1043514000,
'end' => 1043526410,
'coverage' => 14,
}, $store->find_report_source( $job, $sched,
$jan25_2003_noon,
) );
}
sub test_find_report_source {
my $self = $_[0];
my $jan25_2003_noon = timelocal( 0, 5, 12, 25, 0, 2003 );
$self->set_up_test_store();
$self->set_up_report_cfg();
my $store = $self->{'store'};
my $job = new Lire::ReportJob( 'aJob' );
my $sched = new Lire::ReportSchedule( 'weekly', $self->{'test_cfg'} );
$self->assert_deep_equals( { 'source' => 'dlf',
'start' => 1043514000,
'end' => 1043526410,
'coverage' => 2,
}, $store->find_report_source( $job, $sched,
$jan25_2003_noon,
) );
my $dir = "$self->{'store'}{'_store_path'}/daily_reports/aJob/200301";
mkpath( $dir, 0, 0755);
for my $day ( 23, 24, 26, 27 ) {
create_file( "$dir/$day.xml", '' );
}
$self->assert_deep_equals( { 'source' => 'merging',
'start' => 1043298000,
'end' => 1043643600,
'coverage' => 57,
'reports' => [ "$dir/23.xml", "$dir/24.xml",
"$dir/26.xml" ],
'days' => [ '2003-01-23', '2003-01-24',
'2003-01-26' ],
}, $store->find_report_source( $job, $sched,
$jan25_2003_noon,
) );
for my $day ( 19, 20, 21, 22, 25 ) {
create_file( "$dir/$day.xml", '' );
}
$self->assert_deep_equals( { 'source' => 'merging',
'start' => 1043038800,
'end' => 1043643600,
'coverage' => 100,
'reports' => [ "$dir/20.xml", "$dir/21.xml",
"$dir/22.xml", "$dir/23.xml",
"$dir/24.xml", "$dir/25.xml",
"$dir/26.xml" ],
'days' => [ '2003-01-20', '2003-01-21',
'2003-01-22', '2003-01-23',
'2003-01-24', '2003-01-25',
'2003-01-26',
],
}, $store->find_report_source( $job, $sched,
$jan25_2003_noon,
) );
}
sub set_up_plugins {
my $self = $_[0];
$self->set_up_plugin_mgr();
my $mgr = Lire::PluginManager->instance();
$mgr->register_plugin( new tests::helpers::TestDerivedAnalyzer() );
$mgr->register_plugin( new tests::helpers::TestExtendedAnalyzer() );
my $converter = new_proxy Lire::Test::Mock( 'Lire::DlfConverter' );
$converter->set_result( 'name' => 'mydlf',
'title' => 'My DLF',
'schemas' => sub { return ( 'ftp', 'www' ) } );
$mgr->register_plugin( $converter );
my $analyser = new_proxy Lire::Test::Mock( 'Lire::DlfAnalyser' );
$analyser->set_result( 'name' => 'myanalyser',
'title' => 'My DLF',
'description' => '<para>My description</para>',
'src_schema' => 'test-derived',
'dst_schema' => 'test-another' );
$mgr->register_plugin( $analyser );
}
sub set_up_store_config {
my $self = $_[0];
my $config = $self->{'store'}->config();
my $jobspec = $self->{'store'}->config()->spec()->get( 'import_jobs' )->get( 'import_job' );
my $job = $jobspec->instance();
$job->set( $jobspec->get( 'name' )->instance( 'value' => 'myjob' ) );
$job->set( $jobspec->get( 'period' )->instance( 'value' => 'hourly' ) );
$job->set( $jobspec->get( 'service' )->instance( 'value' => 'mydlf' ) );
$job->set( $jobspec->get( 'log_file' )->instance( 'value' => 'test.log' ));
$config->get( 'import_jobs' )->append( $job );
}
sub test_get_stream_config {
my $self = $_[0];
$self->set_up_test_store();
my $store = $self->{'store'};
$self->assert_dies( qr/schema \'wawa\' doesn\'t exist/,
sub { $store->get_stream_config( 'wawa' ) } );
$self->assert_num_equals( 0, scalar $store->{'_config'}->get( 'streams_config' )->elements() );
my $cfg = $store->get_stream_config( 'test-extended' );
$self->assert_isa( 'Lire::Config::Dictionary', $cfg );
$self->assert_str_equals( 'test-extended', $cfg->name() );
$self->assert_isa( 'Lire::Config::DlfStreamSpec', $cfg->spec() );
$self->assert_deep_equals( [ $cfg ],
[ $store->{'_config'}->get( 'streams_config' )->elements() ] );
$self->assert_str_equals( $cfg,
$store->get_stream_config( 'test-extended' ) );
}
sub test_dlf_streams {
my $self = $_[0];
my $store = Lire::DlfStore->open( "$self->{'workingdir'}/test_dlf_streams", 1 );
$store->open_dlf_stream( 'test', 'w' )->close();
$store->open_dlf_stream( 'test-extended', 'w' )->close();
$store->open_dlf_stream( 'test-derived', 'w' )->close();
$self->assert_deep_equals( [ 'test', 'test-derived', 'test-extended' ],
[ sort $store->dlf_streams() ] );
$store->close();
}
sub test_configured_dlf_streams {
my $self = $_[0];
local %Lire::DlfSchema::SCHEMA_CACHE = ( 'www' => 1,
'test-another' => 1 );
$self->set_up_plugins();
$self->set_up_test_store();
$self->set_up_store_config();
$self->assert_deep_equals( [ 'test', 'test-another', 'test-derived',
'test-extended', 'www' ],
[ $self->{'store'}->configured_dlf_streams()] );
my $test_cfg = $self->{'store'}->get_stream_config( 'test' );
$test_cfg->get( 'test-derived' )->set_plugin( 'none' );
$self->assert_deep_equals( [ 'test', 'test-extended', 'www' ],
[ $self->{'store'}->configured_dlf_streams()] );
}
sub set_up_test_clean_streams {
my $self = $_[0];
$self->{'cfg'}{'_lr_config_spec'} = $self->lire_default_config_spec();
my $now = time;
my $one_week_ago = $now - (7 *86400) - 1;
my $yesterday = $now - 86400 - 1;
$self->{'astore'} =
Lire::DlfStore->open( "$self->{'workingdir'}/test_clean_streams", 1 );
my $test = $self->{'astore'}->open_dlf_stream( 'test', 'w' );
$test->write_dlf( { 'time_start' => $one_week_ago } );
$test->write_dlf( { 'time_start' => $yesterday } );
$test->write_dlf( { 'time_start' => $now } );
$test->close();
my $derived = $self->{'astore'}->open_dlf_stream( 'test-derived', 'w' );
$derived->write_dlf( { 'session_start' => $one_week_ago } );
$derived->write_dlf( { 'session_start' => $yesterday } );
$derived->write_dlf( { 'session_start' => $now } );
$derived->close();
$self->{'astore'}->get_stream_config( 'test' )->get( 'keep_days' )->set( 1 );
$self->{'astore'}->get_stream_config( 'test-derived' )->get( 'keep_days' )->set( 0 );
}
sub test_clean_streams {
my $self = $_[0];
$self->set_up_test_clean_streams();
my $test = $self->{'astore'}->open_dlf_stream( 'test', 'w' );
my $derived = $self->{'astore'}->open_dlf_stream( 'test-derived', 'w' );
$self->assert_num_equals( 3, $test->nrecords() );
$self->assert_num_equals( 3, $derived->nrecords() );
$self->{'astore'}->clean_streams();
$self->assert_num_equals( 1, $test->nrecords() );
$self->assert_num_equals( 3, $derived->nrecords() );
}
sub set_up_test_run_analysers {
my $self = $_[0];
$self->set_up_plugin_mgr();
$self->set_up_test_store();
my $mgr = Lire::PluginManager->instance();
$mgr->register_plugin( new tests::helpers::TestDerivedAnalyzer() );
$mgr->register_plugin( new tests::helpers::TestExtendedAnalyzer() );
$self->{'store'}->get_stream_config( 'test' )->get( 'test-extended' )->set_plugin( 'none' );
Lire::Test::Mock->set_mock_factory( 'Lire::DlfAnalyserProcess' );
$self->{'store'} = new_proxy Lire::Test::Mock( $self->{'store'} );
}
sub test_run_analysers {
my $self = $_[0];
$self->set_up_test_run_analysers();
$self->{'store'}->run_analysers( 'test', 'my_source' );
my $analysers =
Lire::Test::Mock->mock_instances( 'Lire::DlfAnalyserProcess' );
$self->assert_num_equals( 1, scalar @$analysers );
$self->assert_str_equals( 'my_source', $analysers->[0]->dlf_source() );
$self->assert_str_equals( $self->{'store'},
$analysers->[0]->dlf_store() );
$self->assert_str_equals( 'derived', $analysers->[0]->dlf_analyser() );
$self->assert_deep_equals( {},
$analysers->[0]->dlf_analyser_config() );
$self->assert_num_equals( 1, $analysers->[0]->invocation_count( 'run_analysis_job' ) );
$self->assert_num_equals( 2, $self->{'store'}->invocation_count( 'run_analysers' ) );
$self->assert_deep_equals( [ $self->{'store'}, 'test-derived',
$analysers->[0]->job_id() ],
$self->{'store'}->get_invocation( 'run_analysers', 1 ) );
}
sub set_up_migrate_report_jobs {
my $self = $_[0];
$self->set_up_plugin_mgr();
my $mgr = Lire::PluginManager->instance();
$mgr->register_plugin( new tests::helpers::TestDerivedAnalyzer() );
$mgr->register_plugin( new tests::helpers::TestExtendedAnalyzer() );
my $analyser = new_proxy Lire::Test::Mock( 'Lire::DlfAnalyser' );
$analyser->set_result( 'name' => 'another',
'title' => 'Another Catogoriser',
'description' => '<para>A fake Categoriser</para>',
'src_schema' => 'test-derived',
'dst_schema' => 'test-extended',
'analyse' => '' );
Lire::PluginManager->register_plugin( $analyser );
my $converter = new_proxy Lire::Test::Mock( 'Lire::DlfConverter' );
$converter->set_result( 'name' => 'test_newapi',
'title' => 'Fake DlfConverter',
'description' => '<para>A fake DlfConverter</para>',
'schemas' => 'test' );
Lire::PluginManager->register_plugin( $converter );
$self->{'cfg'}{'lr_reports_path'} = [ "$self->{'testdir'}/reports" ];
$self->{'cfg'}{'lr_filters_path'} = [ "$self->{'testdir'}/filters" ];
$self->set_up_test_store( 0 );
copy( "$self->{'testdir'}/data/test.cfg", "$self->{'tmpdir'}" );
open my $ofh, "> $self->{'store_path'}/config.xml"
or $self->error( "open >: $!" );
open my $ifh, "$self->{'testdir'}/data/jobs-config-15.xml"
or $self->error( "open <: $!" );
while ( my $line = <$ifh> ) {
$line =~ s!test.cfg!$self->{'tmpdir'}\/test.cfg!;
print $ofh $line;
}
close $ofh;
close $ifh;
$self->set_up_chart_types( 0 );
$self->set_up_output_formats( 0 );
return;
}
sub test_migrate_report_jobs {
my $self = $_[0];
$self->set_up_migrate_report_jobs();
my $store = Lire::DlfStore->open( $self->{'store_path'} );
$store->close();
my $parser = new Lire::Config::Parser( 'spec' => $self->{'cfg'}{'_lr_config_spec'} );
my $expected = $parser->load_config_file( "$self->{'testdir'}/data/jobs-config-15-migrated.xml" );
my $migrated = $parser->load_config_file( "$self->{'store_path'}/config.xml" );
$expected->filename( $migrated->filename() );
$self->assert_deep_equals( $expected, $migrated );
}
1;
syntax highlighted by Code2HTML, v. 0.9.1