|
0
|
1 #!/usr/bin/perl
|
|
|
2 use strict;
|
|
|
3 use warnings;
|
|
|
4
|
|
|
5 use Digest;
|
|
|
6 use File::Path qw(make_path);
|
|
|
7 use IMPL::require {
|
|
|
8 XMLReader => 'XML::LibXML::Reader',
|
|
|
9 UserAgent => 'LWP::UserAgent',
|
|
|
10 Request => 'HTTP::Request',
|
|
|
11 MDParser => 'Yours::Parsers::MDParser',
|
|
|
12 PMDParser => 'Yours::Parsers::PMDParser',
|
|
|
13 Dumper => 'Data::Dumper',
|
|
|
14 Uncompress => 'IO::Uncompress::AnyUncompress'
|
|
|
15 };
|
|
|
16
|
|
|
17 BEGIN {
|
|
|
18 XMLReader->import;
|
|
|
19
|
|
|
20 }
|
|
|
21
|
|
|
22 my %digestTypes = (
|
|
|
23 sha512 => 'SHA-512',
|
|
|
24 sha384 => 'SHA-384',
|
|
|
25 sha256 => 'SHA-256',
|
|
|
26 sha1 => 'SHA-1',
|
|
|
27 md5 => 'MD5'
|
|
|
28 );
|
|
|
29
|
|
|
30 my $repoUrl =
|
|
|
31 'http://download.opensuse.org/repositories/Mono/openSUSE_12.3/';
|
|
|
32 my $repoDir = 'repo';
|
|
1
|
33 my $validate = 0;
|
|
0
|
34
|
|
|
35 my $agent = UserAgent->new();
|
|
|
36 $agent->env_proxy();
|
|
|
37
|
|
|
38 # список файлов, которые должны быть в репозитарии, заполняется по мере загрузки/проверки целостности
|
|
|
39 my %files;
|
|
|
40 # список каталогов, которые должны быть в репозитарии, заполняется по мере загрузки/проверки целостности
|
|
|
41 my %dirs;
|
|
|
42
|
|
|
43 print "loading metadata\n";
|
|
|
44
|
|
|
45 make_path(File::Spec->catdir($repoDir,'repodata')) unless -d File::Spec->catdir($repoDir,'repodata');
|
|
|
46
|
|
|
47 # загружаем основные метаданные
|
|
|
48
|
|
|
49 my $mdLocation = "repodata/repomd.xml";
|
|
|
50 my $mdFile = MakeLocalName($mdLocation);
|
|
|
51
|
|
|
52 my @initial = (
|
|
|
53 $mdLocation,
|
|
|
54 "repodata/repomd.xml.asc",
|
|
|
55 "repodata/repomd.xml.key"
|
|
|
56 );
|
|
|
57
|
|
1
|
58 $files{MakeLocalName($_)} = { location => "${repoUrl}$_" } foreach @initial;
|
|
|
59
|
|
|
60 unless ($validate) {
|
|
|
61 foreach my $initLocation (@initial) {
|
|
|
62 my $file = MakeLocalName($initLocation);
|
|
|
63 my $resp = $agent->get(
|
|
|
64 "${repoUrl}$initLocation",
|
|
|
65 ':content_file' => $file
|
|
|
66 );
|
|
|
67
|
|
|
68 die "failed to load metadata $initLocation: ", $resp->code, " ", $resp->message
|
|
|
69 unless $resp->is_success;
|
|
|
70 }
|
|
0
|
71 }
|
|
|
72
|
|
|
73 my $parser = MDParser->new();
|
|
|
74 $parser->Parse( { location => $mdFile, no_blanks => 1 } );
|
|
|
75
|
|
|
76 # загружаем метаданные о пакетах
|
|
|
77 my %indexMd;
|
|
|
78 foreach my $md (@{$parser->data->{data} || []}) {
|
|
|
79 $indexMd{$md->{type}} = $md;
|
|
|
80
|
|
|
81 print "\t$md->{type}: $md->{location}\n";
|
|
|
82
|
|
|
83 my $file = MakeLocalName($md->{location});
|
|
|
84 $md->{file} = $file;
|
|
|
85 $files{$file} = $md;
|
|
|
86
|
|
|
87 unless (-f $file) {
|
|
|
88 my $resp = $agent->get(
|
|
|
89 "${repoUrl}$md->{location}",
|
|
|
90 ':content_file' => $file
|
|
|
91 );
|
|
|
92 die "failed to load $md->{location}: ", $resp->code, " ", $resp->message
|
|
|
93 unless $resp->is_success;
|
|
|
94 }
|
|
|
95 }
|
|
|
96
|
|
|
97 my $primaryMd = $indexMd{primary}{file};
|
|
|
98
|
|
|
99 my $hdata = Uncompress->new($primaryMd)
|
|
|
100 or die "failed to uncompress $primaryMd";
|
|
|
101
|
|
|
102 print "processing contents\n";
|
|
|
103
|
|
|
104 PMDParser->new(sub {
|
|
2
|
105 my ($package) = @_;
|
|
0
|
106 my $location = $package->{location};
|
|
|
107
|
|
|
108 my $file = MakeLocalName($location);
|
|
|
109 $files{$file} = $package;
|
|
|
110
|
|
|
111
|
|
|
112
|
|
|
113 unless (-f $file) {
|
|
|
114 my $size = sprintf("%0.2fM",$package->{size}/(1024*1024));
|
|
|
115 print "\tfetch $location [${size}]\n";
|
|
|
116
|
|
|
117 $agent->get("${repoUrl}$location", ":content_file" => $file);
|
|
|
118 }
|
|
|
119 })->Parse({ IO => $hdata, no_blanks => 1 });
|
|
|
120
|
|
|
121 print "cleanup\n";
|
|
|
122
|
|
|
123 foreach my $dir (keys %dirs) {
|
|
|
124 print "\t$dir\n";
|
|
|
125 if (opendir(my $hdir, $dir)) {
|
|
|
126 while(my $file = readdir $hdir) {
|
|
|
127 next if $file eq '.' || $file eq '..';
|
|
|
128 my $fullPath = File::Spec->catfile($dir,$file);
|
|
|
129 next unless -f $fullPath;
|
|
|
130
|
|
|
131 unless( $files{$fullPath} ){
|
|
|
132 print "\t\t- $file\n";
|
|
|
133 unlink $fullPath;
|
|
|
134 }
|
|
|
135 }
|
|
|
136 }
|
|
|
137 }
|
|
|
138
|
|
|
139 print "validating\n";
|
|
|
140
|
|
1
|
141 my @bad;
|
|
|
142
|
|
0
|
143 while(my ($file,$md) = each %files) {
|
|
|
144 if (my $checksum = $md->{checksum}) {
|
|
|
145 if( my $type = $digestTypes{lc($checksum->{type})} ) {
|
|
|
146 if(open my $hfile, "<$file") {
|
|
|
147 binmode $hfile;
|
|
|
148 my $digest = Digest->new($type)->addfile($hfile)->hexdigest;
|
|
|
149 next if $digest eq $checksum->{value};
|
|
|
150
|
|
|
151 print "\t$file: $digest ne $checksum->{value}\n";
|
|
|
152 } else {
|
|
|
153 print "\t$file: ", -f $file ? "unable to open" : "missing","\n";
|
|
|
154 }
|
|
1
|
155 push @bad,$md;
|
|
0
|
156 } else {
|
|
|
157 print "\t$file: unknown hash algorithm: $checksum->{type}\n";
|
|
|
158 }
|
|
|
159 }
|
|
|
160 }
|
|
|
161
|
|
1
|
162 print "fixing\n";
|
|
|
163
|
|
|
164 foreach my $md (@bad) {
|
|
|
165 my $location = $md->{location};
|
|
|
166
|
|
|
167 my $file = MakeLocalName($location);
|
|
|
168
|
|
|
169 my $size = sprintf("%0.2fM",$md->{size}/(1024*1024));
|
|
|
170 print "\tfetch $location [${size}]\n";
|
|
|
171
|
|
|
172 $agent->get("${repoUrl}$location", ":content_file" => $file);
|
|
|
173 }
|
|
|
174
|
|
|
175 print "total files: ", scalar keys %files, "\n";
|
|
0
|
176
|
|
|
177 sub MakeLocalName {
|
|
|
178 my ($url) = @_;
|
|
|
179
|
|
|
180 my @parts = split /\//, $url;
|
|
|
181 my $file = pop @parts;
|
|
|
182
|
|
|
183 my $dir = File::Spec->catdir($repoDir,@parts);
|
|
|
184
|
|
|
185 make_path($dir)
|
|
|
186 unless $dirs{$dir};
|
|
|
187 $dirs{$dir} = 1;
|
|
|
188
|
|
|
189 return File::Spec->catfile($dir,$file);
|
|
|
190 } |
