Mercurial > pub > Yours

changeset 2:f2a86753b494

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
implemented object model robust validation
author cin
date Tue, 22 Oct 2013 16:15:22 +0400
parents 30a88ad2b2fd
children ae61af01bfa5
files _test/read_repo.pl _test/repo.pl lib/Yours/FileValidator.pm lib/Yours/Model/Repository.pm lib/Yours/Parsers/PMDParser.pm lib/Yours/SyncRepository.pm
diffstat 6 files changed, 439 insertions(+), 2 deletions(-) [+]
line wrap: on
line diff
--- a/_test/read_repo.pl	Tue Oct 22 05:36:00 2013 +0400
+++ b/_test/read_repo.pl	Tue Oct 22 16:15:22 2013 +0400
@@ -102,7 +102,7 @@
 print "processing contents\n";
 
 PMDParser->new(sub {
-	my ($parser,$package) = @_;
+	my ($package) = @_;
 	my $location = $package->{location};
 	
 	my $file = MakeLocalName($location);
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/_test/repo.pl	Tue Oct 22 16:15:22 2013 +0400
@@ -0,0 +1,32 @@
+#!/usr/bin/perl
+use strict;
+
+$ENV{http_proxy} = "http://10.111.0.3:3128";
+
+use IMPL::require {
+	Repository => 'Yours::Model::Repository',
+	Sync => 'Yours::SyncRepository',
+	Dumper => 'Data::Dumper',
+	
+};
+
+my @repos = (
+	{
+		name => 'gnome',
+		dir => 'gnome',
+		location => 'http://download.opensuse.org/repositories/GNOME:/STABLE:/3.8/openSUSE_12.3/'
+	}
+);
+
+foreach my $info ( @repos ) {
+	eval {
+		my $repo = Repository->new( map $info->{$_},qw(name dir location));
+		Sync
+			->new(*STDOUT) # log to STDOUT
+			->Update($repo);
+	};
+	warn $@ if $@;
+}
+
+
+print "SUCCESS\n";
\ No newline at end of file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/lib/Yours/FileValidator.pm	Tue Oct 22 16:15:22 2013 +0400
@@ -0,0 +1,73 @@
+package Yours::FileValidator;
+use strict;
+
+use Digest;
+use IMPL::declare {
+	base => [
+		'IMPL::Object' => undef
+	]
+};
+
+my %digestTypes = (
+	sha => 'SHA-1',
+	sha512 => 'SHA-512',
+	sha384 => 'SHA-384',
+	sha256 => 'SHA-256',
+	sha1 => 'SHA-1',
+	md5 => 'MD5',
+	md4 => 'MD4'
+);
+
+
+sub Validate {
+	my ($this,$files) = @_;
+	
+	my @bad;
+
+	while(my ($file,$md) = each %$files) {
+		
+		unless (-f $file) {
+			push @bad, {
+				message => "file not found",
+				file => $file,
+				metadata => $md
+			};
+			next;
+		}
+		
+		if (my $checksum = $md->{checksum}) {
+			
+			if( my $type = $digestTypes{lc($checksum->{type})} ) {
+				
+				if(open my $hfile, "<$file") {
+					binmode $hfile;
+					my $digest = Digest->new($type)->addfile($hfile)->hexdigest;
+					next if $digest eq $checksum->{value};
+
+					push @bad, {
+						message => "$digest ne $checksum->{value}",
+						file => $file,
+						metadata => $md
+					};
+				} else {
+					push @bad, {
+						message => "unable to open the file",
+						file => $file,
+						metadata => $md
+					}; 
+				}
+				
+			} else {
+				$this->Log("$file: unknown hash algorithm: $checksum->{type}");
+			}
+		}
+	}
+	
+	return @bad;
+}
+
+sub Log {
+	
+}
+
+1;
\ No newline at end of file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/lib/Yours/Model/Repository.pm	Tue Oct 22 16:15:22 2013 +0400
@@ -0,0 +1,154 @@
+package Yours::Model::Repository;
+use strict;
+
+use File::Spec();
+use IMPL::Exception();
+use IMPL::Const qw(:prop);
+use URI;
+use IMPL::declare {
+	require => {
+		MDParser      => 'Yours::Parsers::MDParser',
+		PMDParser     => 'Yours::Parsers::PMDParser',
+		Uncompress    => 'IO::Uncompress::AnyUncompress',
+		IOException   => '-IMPL::IOException',
+		ArgException  => '-IMPL::InvalidArgumentException',
+		FileValidator => 'Yours::FileValidator'
+	  },
+	  base  => [ 'IMPL::Object' => undef, ],
+	  props => [
+		name      => PROP_RW,
+		localPath => PROP_RW,
+		repoUrl   => PROP_RW
+	  ]
+};
+
+sub CTOR {
+	my ( $this, $name, $localPath, $repoUrl ) = @_;
+
+	die ArgException->new( name => "A name is required" )
+	  unless $name;
+	die ArgException->new( localPath => "A localPath is required" )
+	  unless $localPath;
+	die ArgException->new( repoUrl => "A repoUrl is required" )
+	  unless $repoUrl;
+
+	$this->repoUrl( URI->new($repoUrl) );
+	$this->localPath($localPath);
+	$this->name($name);
+}
+
+sub GetMetadataFiles {
+	my ($this) = @_;
+	
+	return {
+		map {
+			$this->GetLocalFile($_),
+			{
+				location => $this->GetRemoteLocation($_)
+			}
+		} (
+			'repodata/repomd.xml',
+			'repodata/repomd.xml.asc',
+			'repodata/repomd.xml.key'
+		)
+	};
+}
+
+sub GetDescriptionFiles {
+	my ($this) = @_;
+	return {
+		map {
+			$this->GetLocalFile($_->{location}),
+			{
+				location => $this->GetRemoteLocation($_->{location}),
+				checksum => $_->{checksum}
+			}
+		} @{$this->ReadMetadata()->{data} || []}
+	};
+}
+
+sub GetLocalFile {
+	my ( $this, $url ) = @_;
+
+	my @parts = split /\//, $url;
+
+	return File::Spec->catfile( $this->localPath, @parts );
+}
+
+sub GetRemoteLocation {
+	my ( $this, $url ) = @_;
+
+	return URI->new_abs( $url, $this->repoUrl );
+}
+
+sub ReadMetadata {
+	my ($this) = @_;
+
+	my $mdFile = $this->GetLocalFile('repodata/repomd.xml');
+
+	die IOException->new( "File not found", $mdFile )
+	  unless -f $mdFile;
+
+	my $parser = MDParser->new();
+	$parser->Parse( { location => $mdFile, no_blanks => 1 } );
+
+	return $parser->data;
+}
+
+sub ProcessContent {
+	my ( $this, $handler ) = @_;
+	
+	die ArgException->new(handler => 'A handler is required')
+		unless $handler;
+
+	my %index = map { $_->{type}, $_ } @{ $this->ReadMetadata()->{data} || [] };
+
+	my $filePrimary = $this->GetLocalFile( $index{primary}{location} );
+
+	die IOException->new( "File not found", $filePrimary )
+	  unless -f $filePrimary;
+
+	my $hdata = Uncompress->new($filePrimary)
+	  or die IOException->new( "Failed to uncompress file", $filePrimary );
+
+	PMDParser->new(sub {
+		my ($meta) = shift;
+		my $file = $this->GetLocalFile($meta->{location});
+		$meta->{location} = $this->GetRemoteLocation($meta->{location});
+		&$handler($file,$meta);
+	})->Parse( { IO => $hdata, no_blanks => 1 } );
+}
+
+sub ValidateMetadata {
+	my ($this) = @_;
+	
+	my $files = $this->GetMetadataFiles();
+	
+	my $validator = FileValidator->new(); 
+	
+	my @errors = $validator->Validate($files);
+	
+	return @errors if @errors;
+	
+	$files = $this->GetDescriptionFiles;
+	
+	return $validator->Validate($files);
+}
+
+sub ValidateContent {
+	my ($this) = @_;
+	
+	my %files;
+	
+	$this->ProcessContent(sub {
+		my ($meta) = @_;
+		
+		my $file = $this->GetLocalFile($meta->{location});
+		
+		$files{$file} = $meta;
+	});
+	
+	return FileValidator->new()->Validate(\%files);
+}
+
+1;
--- a/lib/Yours/Parsers/PMDParser.pm	Tue Oct 22 05:36:00 2013 +0400
+++ b/lib/Yours/Parsers/PMDParser.pm	Tue Oct 22 16:15:22 2013 +0400
@@ -39,7 +39,7 @@
 		
 		$package->{type} = $type;
 		
-		$me->onpackage->($me,$package)
+		$me->onpackage->($package)
 			if $me->onpackage;
 	});
 }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/lib/Yours/SyncRepository.pm	Tue Oct 22 16:15:22 2013 +0400
@@ -0,0 +1,178 @@
+package Yours::SyncRepository;
+use strict;
+
+use File::Path qw(make_path);
+use File::Spec();
+use IMPL::Const qw(:prop);
+use IMPL::declare {
+	require => {
+		UserAgent => 'LWP::UserAgent',
+		Exception => 'IMPL::Exception',
+		Validator => 'Yours::FileValidator'
+	},
+	base => [
+		'IMPL::Object' => undef,
+		'IMPL::Object::Autofill' => '@_'
+	],
+	props => [
+		agent => PROP_RO,
+		log => PROP_RO,
+		_dirs => PROP_RW,
+		_files => PROP_RW
+	]
+};
+
+sub CTOR {
+	my ($this, $log) = @_;
+	
+	$this->log($log);
+	$this->_dirs({});
+	$this->_files({});
+	
+	my $agent = UserAgent->new();
+	$agent->env_proxy;
+	
+	$this->agent($agent);
+}
+
+sub _RegisterFile {
+	my ($this,$file,$md) = @_;
+	
+	my ($vol,$dir) = File::Spec->splitpath($file);
+	
+	make_path($dir);
+		
+	$this->_dirs->{$dir} = 1;
+	$this->_files->{$file} = $md;
+}
+
+sub _FetchFile {
+	my ($this,$file,$md,$dieOnError) = @_;
+	
+	if ($md->{size}) {
+		$this->Log("    fetch %s  [%.2fM]", $md->{location}, $md->{size}/(1024*1024));
+	} else {
+		$this->Log("    fetch %s", $md->{location});
+	}
+			
+	my $resp = $this->agent->get(
+		$md->{location},
+		':content_file' => $file
+	);
+	
+	unless($resp->is_success) {
+		$this->Log("      %s %s", $resp->code, $resp->message);
+		die Exception->new($resp->code,$resp->message)
+			if $dieOnError; 
+	}
+}
+
+sub Update {
+	my ($this,$repo) = @_;
+	
+	$this->Log("Updating %s [%s]", $repo->name, $repo->repoUrl);
+	
+	$this->Log("  loading metadata");
+	my $files = $repo->GetMetadataFiles;
+	while( my ($file,$md) = each %$files ) {
+		$this->_RegisterFile($file,$md);
+		
+		$this->_FetchFile($file,$md,'die')
+			unless -f $file;
+	}
+	
+	$this->Log("  loading description");
+	$files = $repo->GetDescriptionFiles;
+	while( my ($file,$md) = each %$files ) {
+		$this->_RegisterFile($file,$md);
+		
+		$this->_FetchFile($file,$md,'die')
+			unless -f $file;
+	}
+	
+	my $retry = 3;
+	$this->Log("  validating metadata");
+	while(my @errors = $repo->ValidateMetadata()) {
+		die Exception->new("Unable to complete due errors")
+			unless $retry;
+		$retry--;
+		
+		foreach my $err (@errors) {
+			$this->Log("    %s: %s", $err->{file},$err->{message});
+			$this->_FetchFile($err->{file},$err->{metadata});
+		}
+	}
+	
+	$this->Log("  downloading content");
+	$repo->ProcessContent(sub {
+		my ($file,$md) = @_;
+		
+		$this->_RegisterFile($file,$md);
+		
+		$this->_FetchFile($file,$md,'die')
+			unless -f $file;
+	});
+	
+	$this->Log("  clenup");
+	$this->_Cleanup;
+	
+	$this->Log("  validate");
+	
+	$retry = 3;
+	my $validateFiles = $this->_files;
+	
+	while(my @errors = Validator->new()->Validate($validateFiles)) {
+		die Exception->new("Unable to complete due errors")
+			unless $retry;
+		$retry--;
+		
+		$validateFiles = {};		
+		
+		foreach my $err (@errors) {
+			$this->Log("    %s: %s", $err->{file},$err->{message});
+			
+			$this->_FetchFile($err->{file},$err->{metadata});
+			
+			$validateFiles->{$err->{file}} = $err->{metadata};
+		}
+	}
+	
+	return;
+}
+
+sub _Cleanup {
+	my ($this) = @_;
+	
+	my $dirs = $this->_dirs;
+	my $files = $this->_files; 
+	
+	foreach my $dir (keys %$dirs) {
+		if (opendir(my $hdir, $dir)) {
+			while(my $file = readdir $hdir) {
+				next if $file eq '.' || $file eq '..';
+				my $fullPath = File::Spec->catfile($dir,$file);
+				next unless -f $fullPath;
+				
+				unless( $files->{$fullPath} ){
+					$this->Log("    - %s",$file);
+					unlink $fullPath;
+				}
+			}
+		}
+	}
+}
+
+sub Repair {
+	my ($this) = @_;
+}
+
+sub Log {
+	my ($this,$format,@args) = @_;
+	
+	if (my $h = $this->log) {
+		printf $h ($format,@args);
+		print $h "\n";
+	}
+}
+
+1;
\ No newline at end of file