Annotation of /builds_bin/update_repos

#!/usr/bin/perl

use strict;
use warnings;
use RPM2;
use Data::Dumper;

sub compare($$);

my $releases = 2;

my $releasedir = '/releases';
my $smerel = '7';
my $centosrel = '4';
my $fedorarel = '3';
my $builds = '/builds/rpms';
my $stagedir = '/builds';
my $arch = `uname -i`;
chomp $arch;

my @repos = (
              "$releasedir/$smerel/smeos/$arch/CentOS/RPMS",
              "$releasedir/$smerel/smeos/$arch/SME/RPMS",
              "$releasedir/$smerel/smeos/$arch/RPMS",
              "$releasedir/$smerel/smeos/$arch/SME",
              "$releasedir/$smerel/smeupdates/$arch/RPMS",
              "$releasedir/$smerel/smeupdates-testing/$arch/RPMS",
              "$releasedir/$smerel/smeextras/$arch/RPMS",
              "$releasedir/$smerel/smeaddons/$arch/RPMS",
              "$releasedir/$smerel/smetest/$arch/RPMS",
              "$releasedir/$smerel/smedev/$arch/RPMS",
              "$builds/RPMS/$arch",
              "$builds/RPMS/i586",
              "$builds/RPMS/i686",
              "$builds/RPMS/noarch",
              "$stagedir/RPMS",
            );

my @srcrepos = (
                 "/mirrors/centos/$centosrel/fasttrack/SRPMS",
                 "/mirrors/centos/$centosrel/updates/SRPMS",
                 "/mirrors/centos/$centosrel/os/SRPMS",
                 "/mirrors/centos/$centosrel/extras/SRPMS",
                 "/mirrors/rpmforge/dag/source",
                 "/mirrors/atrpms/src/el$centosrel-$arch/atrpms/stable",
                 "/mirrors/fedora/epel/$centosrel/SRPMS",
                 "/mirrors/fedora/core/updates/$fedorarel/SRPMS",
                 "/mirrors/fedora/core/$fedorarel/source/SRPMS",
                 "/mirrors/fedora/extras/$fedorarel/SRPMS",
                 "$builds/SRPMS",
              );

my %repohash;
my %srcrepohash;
my %changed;
my %copy;
my @remove;

my $rpm_flags = RPM2->vsf_nodsaheader | RPM2->vsf_nodsa;

%{$srcrepohash{$stagedir}} = map { $_ => 0 } grep { /\.rpm$/ } readdir SRPMS if ( opendir SRPMS, "$stagedir/SRPMS" );
closedir SRPMS;

foreach my $repo ( @repos ) {
        if ( $repo =~ m{^($releasedir/$smerel/sme[^/]*)/} ) {
                my $srcrepo = $1;

                %{$srcrepohash{$srcrepo}} = map { $_ => 0 } grep { /\.rpm$/ } readdir SRPMS if ( opendir SRPMS, "$srcrepo/SRPMS" );
                closedir SRPMS;
        }

        opendir RPMS, $repo;
        foreach my $rpm ( grep { m{\.rpm} } readdir RPMS ) {
                my $header = RPM2->open_package("$repo/$rpm", $rpm_flags);
                my $pkg = $header->tag("NAME").".".$header->tag("ARCH");
                if ( $repo =~ m{$stagedir/RPMS} ) {
                        push @{$repohash{$pkg}{$stagedir}}, $header;
                } elsif ( $repo =~ m{$builds/RPMS} ) {
                        push @{$repohash{$pkg}{builds}}, $header;
                } elsif ( $repo =~ m{^($releasedir/$smerel/sme[^/]*)/} ) {
                        my $pos = $1;
                        push @{$repohash{$pkg}{$pos}}, $header;
                }
        }
        closedir RPMS;
}

my %sources;
foreach my $repo ( reverse @srcrepos ) {
        my %temp_repo = map { $_ => $repo } grep { /\.rpm$/ } readdir SRPMS if ( opendir SRPMS, $repo );
        closedir SRPMS;

        %sources = (%sources, %temp_repo);
}

foreach my $pkg ( keys %repohash ) {
        foreach my $repo ( @repos ) {
                if ( $repo =~ m{$stagedir/RPMS} ) {
                        next unless $repohash{$pkg};
                        if ( $repohash{$pkg}{$stagedir} ) {
                                my ($latest, @rest) = sort { $b cmp $a } @{$repohash{$pkg}{$stagedir}};
                                delete $repohash{$pkg}{$stagedir};
                                push @remove, map { $_->filename } @rest if @rest;
                                if ( $repohash{$pkg}{latest} ) {
                                        if ($repohash{$pkg}{latest} ne $latest ) {
                                                push @remove, $latest->filename;
                                                push @{$copy{"$stagedir/RPMS"}}, $repohash{$pkg}{latest}->filename;
                                                $repohash{$pkg}{$stagedir} = $repohash{$pkg}{latest};
                                        } else {
                                                $repohash{$pkg}{$stagedir} = $latest;
                                        }
                                } else {
                                        push @remove, $latest->filename;
                                }
                        } elsif ( $repohash{$pkg}{latest} ) {
                                push @{$copy{"$stagedir/RPMS"}}, $repohash{$pkg}{latest}->filename;
                                $repohash{$pkg}{$stagedir} = $repohash{$pkg}{latest};
                        }
                } elsif ( $repo =~ m{^($releasedir/$smerel/sme[^/]*)/} ) {
                        my $pos = $1;
                        next unless $repohash{$pkg}{$pos};
                        if ( $pos =~ m{/smedev$} ) {
                                next unless $repohash{$pkg}{latest};
                                my @rest = grep { $_ le $repohash{$pkg}{latest} } @{$repohash{$pkg}{$pos}};
                                if ( @rest ) {
                                        push @remove, map { $_->filename } @rest;
                                        $changed{$pos}++;
                                        @{$repohash{$pkg}{$pos}} = grep { $_ gt $repohash{$pkg}{latest} } @{$repohash{$pkg}{$pos}};
                                        delete $repohash{$pkg}{$pos};
                                }
                        } elsif ( ref($repohash{$pkg}{$pos}) eq 'ARRAY' ) {
                                my ($latest, @rest) = sort { $b cmp $a } @{$repohash{$pkg}{$pos}};
                                delete $repohash{$pkg}{$pos};
                                if ( $repohash{$pkg}{latest} && $repohash{$pkg}{latest} ge $latest ) {
                                        push @rest, $latest;
                                } else {
                                        $repohash{$pkg}{latest} = $latest;
                                        $repohash{$pkg}{$pos} = $latest;
                                }
                                if ( @rest ) {
                                        push @remove, map { $_->filename } @rest;
                                        $changed{$pos}++;
                                }
                        }
                }
        }
}

print "\n";
foreach my $pkg ( sort grep { $repohash{$_}{builds} } keys %repohash ) {
        my %versions = ();
        foreach my $ver ( sort { $b cmp $a } @{$repohash{$pkg}{builds}} ) {
                next if $repohash{$pkg}{latest} && $repohash{$pkg}{latest} ge $ver;

                $versions{$ver->tag("VERSION")}++;
                if ( $versions{$ver->tag("VERSION")} > $releases || (scalar keys %versions) > $releases ) {
                        foreach my $rpm ( grep { $ver eq $_ } @{$repohash{$pkg}{"$releasedir/$smerel/smedev"}} ) {
                                push @remove, $rpm->filename;
                                @{$repohash{$pkg}{"$releasedir/$smerel/smedev"}} = grep { $rpm ne $_ } @{$repohash{$pkg}{"$releasedir/$smerel/smedev"}};
                                $changed{"$releasedir/$smerel/smedev"}++;
                        }
                } else {
                        print "Possible update " . $ver->filename . "\n" if $repohash{$pkg}{latest};
                        if ( ! grep { $ver eq $_ } @{$repohash{$pkg}{"$releasedir/$smerel/smedev"}} ) {
                                push @{$copy{"$releasedir/$smerel/smedev/$arch/RPMS"}}, $ver->filename;
                                push @{$repohash{$pkg}{"$releasedir/$smerel/smedev"}}, $ver;
                                $changed{"$releasedir/$smerel/smedev"}++;
                        }
                }
        }
}

foreach my $pkg ( grep { $repohash{$_}{"$releasedir/$smerel/smedev"} } keys %repohash ) {
        foreach my $rpm ( @{$repohash{$pkg}{"$releasedir/$smerel/smedev"}} ) {
                unless ( grep { $rpm eq $_ } @{$repohash{$pkg}{builds}} ) {
                        push @remove, $rpm->filename;
                        @{$repohash{$pkg}{"$releasedir/$smerel/smedev"}} = grep { $rpm ne $_ } @{$repohash{$pkg}{"$releasedir/$smerel/smedev"}};
                        $changed{"$releasedir/$smerel/smedev"}++;
                }
        }
}

print "\n";
foreach my $repo ( sort keys %srcrepohash ) {
        foreach my $pkg ( sort grep { $repohash{$_}{$repo} } keys %repohash ) {
                my @rpms = ref($repohash{$pkg}{$repo}) eq "ARRAY" ? @{$repohash{$pkg}{$repo}} : ($repohash{$pkg}{$repo});
                foreach my $rpm ( sort @rpms ) {
                        my $src;

                        my @sources = ();
                        push @sources, ($src = $rpm->tag("SOURCERPM"));
                        push @sources, $src if (($src = $rpm->tag("SOURCERPM")) =~ s/\.el\d\.rf\./\.rf\./);
                        push @sources, $src if (($src = $rpm->tag("SOURCERPM")) =~ s/\.[0-3]\.el\d+\.rf\./\.rf\./);
                        push @sources, $src if (($src = $rpm->tag("SOURCERPM")) =~ s/\.(el|fc)\d\.at//);
                        push @sources, $src if (($src = $rpm->tag("SOURCERPM")) =~ s/\.at//);
                        push @sources, $src if (($src = $rpm->tag("SOURCERPM")) =~ s/\.(el|fc)\d//);
                        push @sources, $src if (($src = $rpm->tag("SOURCERPM")) =~ s/\.\d+\.(el|fc)\d//);
                        $src = $rpm->tag("SOURCERPM");

                        my $found = 0;
                        foreach my $source ( @sources ) {
                                if ( $sources{$source} ) {
                                        unless ( -f "$repo/SRPMS/$source" ) {
                                                push @{$copy{"$repo/SRPMS"}}, "$sources{$source}/$source";
                                                $changed{$repo}++;
                                        }
                                        $srcrepohash{$repo}{$source}++;
                                        $found++;
                                        last;
                                } elsif ( -f "$repo/SRPMS/$source" ) {
                                        print "Found missing source $repo/SRPMS/$source for ".$rpm->as_nvre()."\n";
                                        $srcrepohash{$repo}{$source}++;
                                        $found++;
                                }
                        }
                        print "Can't find source for ".$rpm->as_nvre()."\n" unless $found;
                }
        }

        foreach my $rpm ( keys %{$srcrepohash{$repo}} ) {
                unless ( $srcrepohash{$repo}{$rpm} ) {
                        push @remove, "$repo/SRPMS/$rpm";
                        $changed{$repo}++;
                }
        }
}

print "\n";
foreach my $repo ( sort keys %copy ) {
        foreach my $rpm ( sort keys %{{ map { $_ => 1 } @{$copy{$repo}} }} ) {
                print "copying $rpm to $repo/\n";
                system(qw(cp --preserve=timestamps), $rpm, "$repo/");
        }
}

print "\n";
foreach my $rpm ( sort keys %{{ map { $_ => 1 } @remove }} ) {
        print "removing $rpm\n";
        unlink $rpm;
}

print "\n";
foreach my $repo ( sort keys %changed ) {
        next if $repo =~ m{^$stagedir};

        my $dir=`readlink -f $repo/$arch`;
        chomp $dir;
        print "rebuilding $dir...\n";
        system(qw(createrepo), "$dir");
        system(qw(repoview), "$dir");
}

1	slords	1.1	#!/usr/bin/perl
2
3			use strict;
4			use warnings;
5			use RPM2;
6			use Data::Dumper;
7
8			sub compare($$);
9
10			my $releases = 2;
11
12	slords	1.2	my $releasedir = '/releases';
13			my $smerel = '7';
14			my $centosrel = '4';
15			my $fedorarel = '3';
16			my $builds = '/builds/rpms';
17	slords	1.1	my $stagedir = '/builds';
18	slords	1.9	my $arch = `uname -i`;
19			chomp $arch;
20	slords	1.2
21			my @repos = (
22			"$releasedir/$smerel/smeos/$arch/CentOS/RPMS",
23	slords	1.5	"$releasedir/$smerel/smeos/$arch/SME/RPMS",
24			"$releasedir/$smerel/smeos/$arch/RPMS",
25			"$releasedir/$smerel/smeos/$arch/SME",
26	slords	1.2	"$releasedir/$smerel/smeupdates/$arch/RPMS",
27			"$releasedir/$smerel/smeupdates-testing/$arch/RPMS",
28			"$releasedir/$smerel/smeextras/$arch/RPMS",
29			"$releasedir/$smerel/smeaddons/$arch/RPMS",
30			"$releasedir/$smerel/smetest/$arch/RPMS",
31			"$releasedir/$smerel/smedev/$arch/RPMS",
32			"$builds/RPMS/$arch",
33			"$builds/RPMS/i586",
34			"$builds/RPMS/i686",
35			"$builds/RPMS/noarch",
36			"$stagedir/RPMS",
37			);
38
39			my @srcrepos = (
40			"/mirrors/centos/$centosrel/fasttrack/SRPMS",
41			"/mirrors/centos/$centosrel/updates/SRPMS",
42			"/mirrors/centos/$centosrel/os/SRPMS",
43			"/mirrors/centos/$centosrel/extras/SRPMS",
44			"/mirrors/rpmforge/dag/source",
45			"/mirrors/atrpms/src/el$centosrel-$arch/atrpms/stable",
46	slords	1.4	"/mirrors/fedora/epel/$centosrel/SRPMS",
47	slords	1.2	"/mirrors/fedora/core/updates/$fedorarel/SRPMS",
48			"/mirrors/fedora/core/$fedorarel/source/SRPMS",
49			"/mirrors/fedora/extras/$fedorarel/SRPMS",
50			"$builds/SRPMS",
51			);
52	slords	1.1
53			my %repohash;
54			my %srcrepohash;
55			my %changed;
56			my %copy;
57			my @remove;
58
59	slords	1.2	my $rpm_flags = RPM2->vsf_nodsaheader \| RPM2->vsf_nodsa;
60
61	slords	1.1	%{$srcrepohash{$stagedir}} = map { $_ => 0 } grep { /\.rpm$/ } readdir SRPMS if ( opendir SRPMS, "$stagedir/SRPMS" );
62			closedir SRPMS;
63
64			foreach my $repo ( @repos ) {
65	slords	1.2	if ( $repo =~ m{^($releasedir/$smerel/sme[^/]*)/} ) {
66			my $srcrepo = $1;
67	slords	1.1
68	slords	1.2	%{$srcrepohash{$srcrepo}} = map { $_ => 0 } grep { /\.rpm$/ } readdir SRPMS if ( opendir SRPMS, "$srcrepo/SRPMS" );
69			closedir SRPMS;
70			}
71
72			opendir RPMS, $repo;
73			foreach my $rpm ( grep { m{\.rpm} } readdir RPMS ) {
74			my $header = RPM2->open_package("$repo/$rpm", $rpm_flags);
75			my $pkg = $header->tag("NAME").".".$header->tag("ARCH");
76			if ( $repo =~ m{$stagedir/RPMS} ) {
77			push @{$repohash{$pkg}{$stagedir}}, $header;
78			} elsif ( $repo =~ m{$builds/RPMS} ) {
79			push @{$repohash{$pkg}{builds}}, $header;
80			} elsif ( $repo =~ m{^($releasedir/$smerel/sme[^/]*)/} ) {
81			my $pos = $1;
82			push @{$repohash{$pkg}{$pos}}, $header;
83			}
84			}
85			closedir RPMS;
86	slords	1.1	}
87
88			my %sources;
89			foreach my $repo ( reverse @srcrepos ) {
90	slords	1.2	my %temp_repo = map { $_ => $repo } grep { /\.rpm$/ } readdir SRPMS if ( opendir SRPMS, $repo );
91			closedir SRPMS;
92	slords	1.1
93	slords	1.2	%sources = (%sources, %temp_repo);
94	slords	1.1	}
95
96			foreach my $pkg ( keys %repohash ) {
97	slords	1.2	foreach my $repo ( @repos ) {
98			if ( $repo =~ m{$stagedir/RPMS} ) {
99			next unless $repohash{$pkg};
100			if ( $repohash{$pkg}{$stagedir} ) {
101			my ($latest, @rest) = sort { $b cmp $a } @{$repohash{$pkg}{$stagedir}};
102			delete $repohash{$pkg}{$stagedir};
103			push @remove, map { $_->filename } @rest if @rest;
104			if ( $repohash{$pkg}{latest} ) {
105			if ($repohash{$pkg}{latest} ne $latest ) {
106			push @remove, $latest->filename;
107			push @{$copy{"$stagedir/RPMS"}}, $repohash{$pkg}{latest}->filename;
108			$repohash{$pkg}{$stagedir} = $repohash{$pkg}{latest};
109			} else {
110			$repohash{$pkg}{$stagedir} = $latest;
111			}
112			} else {
113			push @remove, $latest->filename;
114			}
115			} elsif ( $repohash{$pkg}{latest} ) {
116			push @{$copy{"$stagedir/RPMS"}}, $repohash{$pkg}{latest}->filename;
117			$repohash{$pkg}{$stagedir} = $repohash{$pkg}{latest};
118			}
119			} elsif ( $repo =~ m{^($releasedir/$smerel/sme[^/]*)/} ) {
120			my $pos = $1;
121			next unless $repohash{$pkg}{$pos};
122			if ( $pos =~ m{/smedev$} ) {
123			next unless $repohash{$pkg}{latest};
124			my @rest = grep { $_ le $repohash{$pkg}{latest} } @{$repohash{$pkg}{$pos}};
125			if ( @rest ) {
126			push @remove, map { $_->filename } @rest;
127			$changed{$pos}++;
128			@{$repohash{$pkg}{$pos}} = grep { $_ gt $repohash{$pkg}{latest} } @{$repohash{$pkg}{$pos}};
129			delete $repohash{$pkg}{$pos};
130			}
131	slords	1.6	} elsif ( ref($repohash{$pkg}{$pos}) eq 'ARRAY' ) {
132	slords	1.2	my ($latest, @rest) = sort { $b cmp $a } @{$repohash{$pkg}{$pos}};
133			delete $repohash{$pkg}{$pos};
134			if ( $repohash{$pkg}{latest} && $repohash{$pkg}{latest} ge $latest ) {
135			push @rest, $latest;
136			} else {
137			$repohash{$pkg}{latest} = $latest;
138			$repohash{$pkg}{$pos} = $latest;
139			}
140			if ( @rest ) {
141			push @remove, map { $_->filename } @rest;
142			$changed{$pos}++;
143			}
144			}
145			}
146			}
147	slords	1.1	}
148
149	slords	1.2	print "\n";
150	slords	1.1	foreach my $pkg ( sort grep { $repohash{$_}{builds} } keys %repohash ) {
151	slords	1.2	my %versions = ();
152			foreach my $ver ( sort { $b cmp $a } @{$repohash{$pkg}{builds}} ) {
153			next if $repohash{$pkg}{latest} && $repohash{$pkg}{latest} ge $ver;
154
155			$versions{$ver->tag("VERSION")}++;
156			if ( $versions{$ver->tag("VERSION")} > $releases \|\| (scalar keys %versions) > $releases ) {
157			foreach my $rpm ( grep { $ver eq $_ } @{$repohash{$pkg}{"$releasedir/$smerel/smedev"}} ) {
158			push @remove, $rpm->filename;
159			@{$repohash{$pkg}{"$releasedir/$smerel/smedev"}} = grep { $rpm ne $_ } @{$repohash{$pkg}{"$releasedir/$smerel/smedev"}};
160			$changed{"$releasedir/$smerel/smedev"}++;
161			}
162			} else {
163			print "Possible update " . $ver->filename . "\n" if $repohash{$pkg}{latest};
164			if ( ! grep { $ver eq $_ } @{$repohash{$pkg}{"$releasedir/$smerel/smedev"}} ) {
165			push @{$copy{"$releasedir/$smerel/smedev/$arch/RPMS"}}, $ver->filename;
166			push @{$repohash{$pkg}{"$releasedir/$smerel/smedev"}}, $ver;
167			$changed{"$releasedir/$smerel/smedev"}++;
168			}
169			}
170			}
171			}
172
173			foreach my $pkg ( grep { $repohash{$_}{"$releasedir/$smerel/smedev"} } keys %repohash ) {
174			foreach my $rpm ( @{$repohash{$pkg}{"$releasedir/$smerel/smedev"}} ) {
175			unless ( grep { $rpm eq $_ } @{$repohash{$pkg}{builds}} ) {
176			push @remove, $rpm->filename;
177			@{$repohash{$pkg}{"$releasedir/$smerel/smedev"}} = grep { $rpm ne $_ } @{$repohash{$pkg}{"$releasedir/$smerel/smedev"}};
178			$changed{"$releasedir/$smerel/smedev"}++;
179			}
180			}
181	slords	1.1	}
182
183			print "\n";
184			foreach my $repo ( sort keys %srcrepohash ) {
185	slords	1.2	foreach my $pkg ( sort grep { $repohash{$_}{$repo} } keys %repohash ) {
186			my @rpms = ref($repohash{$pkg}{$repo}) eq "ARRAY" ? @{$repohash{$pkg}{$repo}} : ($repohash{$pkg}{$repo});
187			foreach my $rpm ( sort @rpms ) {
188	slords	1.9	my $src;
189	slords	1.2
190			my @sources = ();
191	slords	1.9	push @sources, ($src = $rpm->tag("SOURCERPM"));
192			push @sources, $src if (($src = $rpm->tag("SOURCERPM")) =~ s/\.el\d\.rf\./\.rf\./);
193			push @sources, $src if (($src = $rpm->tag("SOURCERPM")) =~ s/\.[0-3]\.el\d+\.rf\./\.rf\./);
194			push @sources, $src if (($src = $rpm->tag("SOURCERPM")) =~ s/\.(el\|fc)\d\.at//);
195			push @sources, $src if (($src = $rpm->tag("SOURCERPM")) =~ s/\.at//);
196			push @sources, $src if (($src = $rpm->tag("SOURCERPM")) =~ s/\.(el\|fc)\d//);
197	slords	1.10	push @sources, $src if (($src = $rpm->tag("SOURCERPM")) =~ s/\.\d+\.(el\|fc)\d//);
198	slords	1.9	$src = $rpm->tag("SOURCERPM");
199	slords	1.2
200			my $found = 0;
201			foreach my $source ( @sources ) {
202			if ( $sources{$source} ) {
203			unless ( -f "$repo/SRPMS/$source" ) {
204			push @{$copy{"$repo/SRPMS"}}, "$sources{$source}/$source";
205			$changed{$repo}++;
206			}
207			$srcrepohash{$repo}{$source}++;
208			$found++;
209			last;
210			} elsif ( -f "$repo/SRPMS/$source" ) {
211			print "Found missing source $repo/SRPMS/$source for ".$rpm->as_nvre()."\n";
212			$srcrepohash{$repo}{$source}++;
213			$found++;
214			}
215			}
216			print "Can't find source for ".$rpm->as_nvre()."\n" unless $found;
217			}
218			}
219
220			foreach my $rpm ( keys %{$srcrepohash{$repo}} ) {
221			unless ( $srcrepohash{$repo}{$rpm} ) {
222			push @remove, "$repo/SRPMS/$rpm";
223			$changed{$repo}++;
224			}
225			}
226	slords	1.1	}
227
228			print "\n";
229			foreach my $repo ( sort keys %copy ) {
230	slords	1.2	foreach my $rpm ( sort keys %{{ map { $_ => 1 } @{$copy{$repo}} }} ) {
231			print "copying $rpm to $repo/\n";
232			system(qw(cp --preserve=timestamps), $rpm, "$repo/");
233			}
234	slords	1.1	}
235
236			print "\n";
237			foreach my $rpm ( sort keys %{{ map { $_ => 1 } @remove }} ) {
238	slords	1.2	print "removing $rpm\n";
239			unlink $rpm;
240	slords	1.1	}
241
242			print "\n";
243			foreach my $repo ( sort keys %changed ) {
244	slords	1.2	next if $repo =~ m{^$stagedir};
245	slords	1.1
246	slords	1.7	my $dir=`readlink -f $repo/$arch`;
247			chomp $dir;
248			print "rebuilding $dir...\n";
249			system(qw(createrepo), "$dir");
250			system(qw(repoview), "$dir");
251	slords	1.1	}
252	slords	1.2