Annotation of /builds_bin/update_repos

#!/usr/bin/perl

use strict;
use warnings;
use RPM2;
use Data::Dumper;

sub compare($$);

my $releases = 2;

my $releasedir = '/releases';
my $smerel = '7';
my $centosrel = '4';
my $fedorarel = '3';
my $builds = '/builds/rpms';
my $stagedir = '/builds';
my $arch = `uname -i`;
chomp $arch;

my @repos = (
              "$releasedir/$smerel/smeos/$arch/SME",
              "$releasedir/$smerel/smeos/$arch/SME/RPMS",
              "$releasedir/$smerel/smeos/$arch/CentOS/RPMS",
              "$releasedir/$smerel/smeupdates/$arch/RPMS",
              "$releasedir/$smerel/smeupdates-testing/$arch/RPMS",
              "$releasedir/$smerel/smeextras/$arch/RPMS",
              "$releasedir/$smerel/smeaddons/$arch/RPMS",
              "$releasedir/$smerel/smecontribs/$arch/RPMS",
              "$releasedir/$smerel/smetest/$arch/RPMS",
              "$releasedir/$smerel/smedev/$arch/RPMS",
              "$builds/RPMS/$arch",
              "$builds/RPMS/i586",
              "$builds/RPMS/i686",
              "$builds/RPMS/noarch",
              "$stagedir/RPMS-$arch",
            );

my @srcrepos = (
                 "/mirrors/centos/$centosrel/fasttrack/SRPMS",
                 "/mirrors/centos/$centosrel/updates/SRPMS",
                 "/mirrors/centos/$centosrel/os/SRPMS",
                 "/mirrors/centos/$centosrel/extras/SRPMS",
                 "/mirrors/rpmforge/dag/source",
                 "/mirrors/atrpms/src/el$centosrel-$arch/atrpms/stable",
                 "/mirrors/atrpms/src/el$centosrel-$arch/atrpms/testing",
                 "/mirrors/fedora/epel/$centosrel/SRPMS",
                 "/mirrors/fedora/core/updates/$fedorarel/SRPMS",
                 "/mirrors/fedora/core/$fedorarel/source/SRPMS",
                 "/mirrors/fedora/extras/$fedorarel/SRPMS",
                 "$builds/SRPMS",
              );

my $rpm_flags = RPM2->vsf_nodsaheader | RPM2->vsf_nodsa;

my @rpms;
my %srpms;
my %sources;
my %changed;
my %copy;

print "\nReading repos...\n";
foreach my $repo ( @repos ) {
        my $r = $repo;
        if ( $repo =~ m{$stagedir/RPMS} ) {
                $r = $stagedir;
        } elsif ( $repo =~ m{$builds/RPMS} ) {
                $r = 'builds';
        } elsif ( $repo =~ m{^($releasedir/$smerel/sme[^/]*)/} ) {
                $r = $1;
        }

        if ( $r eq $stagedir ) {
                opendir SRPMS, "$r/SRPMS-$arch" or next;
                foreach my $srpm ( grep { /\.rpm$/ } readdir SRPMS ) {
                        $srpms{$r}{$srpm} = 0;
                        $sources{$srpm} = "$r/SRPMS-$arch";
                }
                closedir SRPMS;
        } elsif ( $r ne 'builds' ) {
                opendir SRPMS, "$r/SRPMS" or next;
                foreach my $srpm ( grep { /\.rpm$/ } readdir SRPMS ) {
                        $srpms{$r}{$srpm} = 0;
                        $sources{$srpm} = "$r/SRPMS";
                }
                closedir SRPMS;
        }

        opendir RPMS, $repo or next;
        print " - $repo\n";
        foreach my $pkg ( reverse sort map { RPM2->open_package("$repo/$_", $rpm_flags); } grep { /\.rpm$/ } readdir RPMS ) {
                my $rpm = { name => $pkg->tag("NAME"),
                                filename => join("-", map { $pkg->tag($_) } qw/name version release/) . "." . $pkg->tag("ARCH"),
                                version => $pkg->tag("VERSION"),
                                release => $pkg->tag("RELEASE"),
                                repo => $r,
                                rpm => $pkg,
                              };
                my $source = $pkg->tag("SOURCERPM");
                $source =~ s#-$rpm->{version}-$rpm->{release}.*##;
                $rpm->{source} = $source;

                if ( $rpm->{repo} =~ m{^$releasedir/$smerel/} && $rpm->{repo} !~ m{/smedev$} ) {
                        $rpm->{repo} = 'deleted' if grep { $_->{repo} ne $rpm->{repo} && $_->{name} eq $rpm->{name} && $_->{rpm} ge $rpm->{rpm} } @rpms;
                        $rpm->{repo} = 'deleted' if grep { $_->{repo} eq $rpm->{repo} && $_->{name} eq $rpm->{name} && $_->{rpm} gt $rpm->{rpm} } @rpms;
                        $changed{$r}++ if $rpm->{repo} eq 'deleted';
                }

                push @rpms, $rpm;
        }
        closedir RPMS;
}

print "\nReading source repos...\n";
foreach my $repo ( reverse @srcrepos ) {
        my %temp_repo = map { $_ => $repo } grep { /\.rpm$/ } readdir SRPMS if ( opendir SRPMS, $repo );
        closedir SRPMS;

        %sources = (%sources, %temp_repo);
}

print "\nFinding latest packages...\n";
my @latest;
foreach my $rpm ( reverse grep { $_->{repo} =~ m{^$releasedir/$smerel/} && $_->{repo} !~ m{/sme(addons|contribs|dev|extras)$} } @rpms )  {
        unless ( grep { $_->{name} eq $rpm->{name} && $_->{rpm} gt $rpm->{rpm} } @latest ) {
                push @latest, $rpm;
                unless ( grep { $_->{repo} eq $stagedir && $_->{filename} eq $rpm->{filename} } @rpms ) {
                        push @{$copy{"$stagedir/RPMS-$arch"}}, $rpm->{rpm}->filename;
                        push @rpms, { name => $rpm->{name},
                                      filename => $rpm->{filename},
                                      version => $rpm->{version},
                                      release => $rpm->{release},
                                      repo => $stagedir,
                                      rpm => $rpm->{rpm},
                                      source => $rpm->{source}
                                    };
                }
        }
}

foreach my $rpm ( grep { $_->{repo} eq $stagedir } @rpms )  {
        $rpm->{repo} = 'deleted' unless grep { $_->{filename} eq $rpm->{filename} } @latest;
}

foreach my $rpm ( reverse grep { $_->{repo} =~ m{^$releasedir/$smerel/} && $_->{repo} =~ m{/sme(addons|contribs|extras)$} } @rpms )  {
        unless ( grep { $_->{name} eq $rpm->{name} && $_->{rpm} gt $rpm->{rpm} } @latest ) {
                push @latest, $rpm;
        }
}

print "\nFinding development packages...\n";
my @devel;
my %versions;
foreach my $rpm ( grep { $_->{repo} eq 'builds' } @rpms )  {
        next if grep { $_->{name} eq $rpm->{name} && $_->{rpm} ge $rpm->{rpm} } @latest;

        $versions{$rpm->{name}}{$rpm->{version}}++ unless grep { $_->{name} eq $rpm->{name} && $_->{rpm} eq $rpm->{rpm} } @devel;
        next if $versions{$rpm->{name}}{$rpm->{version}} > $releases || scalar keys %{$versions{$rpm->{name}}} > $releases;
        print "Possible update " . $rpm->{rpm}->filename . "\n" if grep { $_->{name} eq $rpm->{name} } @latest;

        push @devel, $rpm;
        unless ( grep { $_->{repo} =~ m{/smedev$} && $_->{filename} eq $rpm->{filename} } @rpms ) {
                push @{$copy{"$releasedir/$smerel/smedev/$arch/RPMS"}}, $rpm->{rpm}->filename;
                $changed{"$releasedir/$smerel/smedev"}++;
                push @rpms, { name => $rpm->{name},
                              filename => $rpm->{filename},
                              version => $rpm->{version},
                              release => $rpm->{release},
                              repo => "$releasedir/$smerel/smedev",
                              rpm => $rpm->{rpm},
                              source => $rpm->{source}
                            };
        }
}

foreach my $rpm ( grep { $_->{repo} =~ m{/smedev$} } @rpms )  {
        $rpm->{repo} = 'deleted' unless grep { $_->{filename} eq $rpm->{filename} } @devel;
}

print "\nFinding source packages...\n";
foreach my $rpm ( grep { $_->{repo} !~ m{^(builds|deleted)$} } @rpms ) {
        my $src;
        my $srpm = $rpm->{rpm}->tag("SOURCERPM");

        my @sources = ();
        push @sources, ($src = $srpm);
        push @sources, $src if (($src = $srpm) =~ s/\.el\d\.rf\./\.rf\./);
        push @sources, $src if (($src = $srpm) =~ s/\.[0-3]\.el\d+\.rf\./\.rf\./);
        push @sources, $src if (($src = $srpm) =~ s/\.(el|fc)\d\.at//);
        push @sources, $src if (($src = $srpm) =~ s/\.at//);
        push @sources, $src if (($src = $srpm) =~ s/\.(el|fc)\d//);
        push @sources, $src if (($src = $srpm) =~ s/\.\d+\.(el|fc)\d//);

        my $found = 0;
        my $repo = $rpm->{repo};
        foreach my $source ( @sources ) {
                if ( $sources{$source} ) {
                        if ($repo eq $stagedir) {
                                push @{$copy{"$repo/SRPMS-$arch"}}, "$sources{$source}/$source" unless -f "$repo/SRPMS-$arch/$source";
                        } else {
                                unless ( -f "$repo/SRPMS/$source" ) {
                                        push @{$copy{"$repo/SRPMS"}}, "$sources{$source}/$source";
                                        $changed{$repo}++;
                                }
                        }
                        $srpms{$repo}{$source}++;
                        $found++;
                        last;
                }
        }
        print "Can't find source for ".$rpm->{rpm}->filename."\n" unless $found;
}

my @remove;
foreach my $repo ( keys %srpms ) {
        foreach my $rpm ( keys %{$srpms{$repo}} ) {
                unless ( $srpms{$repo}{$rpm} ) {
                        if ($repo eq $stagedir) {
                                push @remove, "$repo/SRPMS-$arch/$rpm";
                        } else {
                                push @remove, "$repo/SRPMS/$rpm";
                                $changed{$repo}++;
                        }
                }
        }
}

print "\n";
foreach my $repo ( sort keys %copy ) {
        foreach my $rpm ( keys %{{ map { $_ => 1 } @{$copy{$repo}} }} ) {
                print "copying $rpm to $repo/\n";
                mkdir "$repo" unless -d "$repo";
                system(qw(cp --preserve=timestamps), $rpm, "$repo/");
        }
}

print "\n";
foreach my $rpm ( keys %{{ map { $_ => 1 } (@remove, map { $_->{rpm}->filename } grep { $_->{repo} eq 'deleted' } @rpms) }} ) {
        print "removing $rpm\n";
        unlink $rpm;
}

print "\n";
foreach my $repo ( sort keys %changed ) {
        my $dir=`readlink -f $repo/$arch`;
        chomp $dir;
        print "rebuilding $dir...\n";
        system(qw(createrepo), "$dir");
        system(qw(rm -rf), "$dir/.olddata") if -d "$dir/.olddata";
        system(qw(repoview), "$dir");
        system(qw(rm -rf), "$dir/.repodata") if "$dir/.repodata";
}
1	slords	1.1	#!/usr/bin/perl
2
3			use strict;
4			use warnings;
5			use RPM2;
6			use Data::Dumper;
7
8			sub compare($$);
9
10			my $releases = 2;
11
12	slords	1.2	my $releasedir = '/releases';
13			my $smerel = '7';
14			my $centosrel = '4';
15			my $fedorarel = '3';
16			my $builds = '/builds/rpms';
17	slords	1.1	my $stagedir = '/builds';
18	slords	1.9	my $arch = `uname -i`;
19			chomp $arch;
20	slords	1.2
21			my @repos = (
22	slords	1.18	"$releasedir/$smerel/smeos/$arch/SME",
23			"$releasedir/$smerel/smeos/$arch/SME/RPMS",
24	slords	1.2	"$releasedir/$smerel/smeos/$arch/CentOS/RPMS",
25			"$releasedir/$smerel/smeupdates/$arch/RPMS",
26			"$releasedir/$smerel/smeupdates-testing/$arch/RPMS",
27			"$releasedir/$smerel/smeextras/$arch/RPMS",
28			"$releasedir/$smerel/smeaddons/$arch/RPMS",
29	slords	1.14	"$releasedir/$smerel/smecontribs/$arch/RPMS",
30	slords	1.2	"$releasedir/$smerel/smetest/$arch/RPMS",
31			"$releasedir/$smerel/smedev/$arch/RPMS",
32			"$builds/RPMS/$arch",
33			"$builds/RPMS/i586",
34			"$builds/RPMS/i686",
35			"$builds/RPMS/noarch",
36	slords	1.18	"$stagedir/RPMS-$arch",
37	slords	1.2	);
38
39			my @srcrepos = (
40			"/mirrors/centos/$centosrel/fasttrack/SRPMS",
41			"/mirrors/centos/$centosrel/updates/SRPMS",
42			"/mirrors/centos/$centosrel/os/SRPMS",
43			"/mirrors/centos/$centosrel/extras/SRPMS",
44			"/mirrors/rpmforge/dag/source",
45			"/mirrors/atrpms/src/el$centosrel-$arch/atrpms/stable",
46	slords	1.13	"/mirrors/atrpms/src/el$centosrel-$arch/atrpms/testing",
47	slords	1.4	"/mirrors/fedora/epel/$centosrel/SRPMS",
48	slords	1.2	"/mirrors/fedora/core/updates/$fedorarel/SRPMS",
49			"/mirrors/fedora/core/$fedorarel/source/SRPMS",
50			"/mirrors/fedora/extras/$fedorarel/SRPMS",
51			"$builds/SRPMS",
52			);
53	slords	1.1
54	slords	1.11	my $rpm_flags = RPM2->vsf_nodsaheader \| RPM2->vsf_nodsa;
55
56			my @rpms;
57			my %srpms;
58			my %sources;
59	slords	1.1	my %changed;
60			my %copy;
61
62	slords	1.18	print "\nReading repos...\n";
63	slords	1.1	foreach my $repo ( @repos ) {
64	slords	1.11	my $r = $repo;
65			if ( $repo =~ m{$stagedir/RPMS} ) {
66			$r = $stagedir;
67			} elsif ( $repo =~ m{$builds/RPMS} ) {
68			$r = 'builds';
69			} elsif ( $repo =~ m{^($releasedir/$smerel/sme[^/]*)/} ) {
70			$r = $1;
71			}
72
73	slords	1.18	if ( $r eq $stagedir ) {
74			opendir SRPMS, "$r/SRPMS-$arch" or next;
75			foreach my $srpm ( grep { /\.rpm$/ } readdir SRPMS ) {
76			$srpms{$r}{$srpm} = 0;
77			$sources{$srpm} = "$r/SRPMS-$arch";
78			}
79			closedir SRPMS;
80			} elsif ( $r ne 'builds' ) {
81	slords	1.11	opendir SRPMS, "$r/SRPMS" or next;
82			foreach my $srpm ( grep { /\.rpm$/ } readdir SRPMS ) {
83			$srpms{$r}{$srpm} = 0;
84			$sources{$srpm} = "$r/SRPMS";
85			}
86			closedir SRPMS;
87			}
88
89			opendir RPMS, $repo or next;
90	slords	1.18	print " - $repo\n";
91	slords	1.11	foreach my $pkg ( reverse sort map { RPM2->open_package("$repo/$_", $rpm_flags); } grep { /\.rpm$/ } readdir RPMS ) {
92			my $rpm = { name => $pkg->tag("NAME"),
93			filename => join("-", map { $pkg->tag($_) } qw/name version release/) . "." . $pkg->tag("ARCH"),
94			version => $pkg->tag("VERSION"),
95			release => $pkg->tag("RELEASE"),
96			repo => $r,
97			rpm => $pkg,
98			};
99			my $source = $pkg->tag("SOURCERPM");
100			$source =~ s#-$rpm->{version}-$rpm->{release}.*##;
101			$rpm->{source} = $source;
102
103			if ( $rpm->{repo} =~ m{^$releasedir/$smerel/} && $rpm->{repo} !~ m{/smedev$} ) {
104			$rpm->{repo} = 'deleted' if grep { $_->{repo} ne $rpm->{repo} && $_->{name} eq $rpm->{name} && $_->{rpm} ge $rpm->{rpm} } @rpms;
105			$rpm->{repo} = 'deleted' if grep { $_->{repo} eq $rpm->{repo} && $_->{name} eq $rpm->{name} && $_->{rpm} gt $rpm->{rpm} } @rpms;
106			$changed{$r}++ if $rpm->{repo} eq 'deleted';
107			}
108
109			push @rpms, $rpm;
110			}
111			closedir RPMS;
112	slords	1.1	}
113
114	slords	1.18	print "\nReading source repos...\n";
115	slords	1.1	foreach my $repo ( reverse @srcrepos ) {
116	slords	1.2	my %temp_repo = map { $_ => $repo } grep { /\.rpm$/ } readdir SRPMS if ( opendir SRPMS, $repo );
117			closedir SRPMS;
118	slords	1.1
119	slords	1.2	%sources = (%sources, %temp_repo);
120	slords	1.1	}
121
122	slords	1.18	print "\nFinding latest packages...\n";
123	slords	1.11	my @latest;
124	slords	1.14	foreach my $rpm ( reverse grep { $_->{repo} =~ m{^$releasedir/$smerel/} && $_->{repo} !~ m{/sme(addons\|contribs\|dev\|extras)$} } @rpms ) {
125	slords	1.11	unless ( grep { $_->{name} eq $rpm->{name} && $_->{rpm} gt $rpm->{rpm} } @latest ) {
126			push @latest, $rpm;
127			unless ( grep { $_->{repo} eq $stagedir && $_->{filename} eq $rpm->{filename} } @rpms ) {
128	slords	1.18	push @{$copy{"$stagedir/RPMS-$arch"}}, $rpm->{rpm}->filename;
129	slords	1.11	push @rpms, { name => $rpm->{name},
130			filename => $rpm->{filename},
131			version => $rpm->{version},
132			release => $rpm->{release},
133			repo => $stagedir,
134			rpm => $rpm->{rpm},
135			source => $rpm->{source}
136			};
137			}
138			}
139			}
140
141			foreach my $rpm ( grep { $_->{repo} eq $stagedir } @rpms ) {
142			$rpm->{repo} = 'deleted' unless grep { $_->{filename} eq $rpm->{filename} } @latest;
143			}
144
145	slords	1.14	foreach my $rpm ( reverse grep { $_->{repo} =~ m{^$releasedir/$smerel/} && $_->{repo} =~ m{/sme(addons\|contribs\|extras)$} } @rpms ) {
146			unless ( grep { $_->{name} eq $rpm->{name} && $_->{rpm} gt $rpm->{rpm} } @latest ) {
147			push @latest, $rpm;
148			}
149			}
150
151	slords	1.18	print "\nFinding development packages...\n";
152	slords	1.11	my @devel;
153			my %versions;
154			foreach my $rpm ( grep { $_->{repo} eq 'builds' } @rpms ) {
155			next if grep { $_->{name} eq $rpm->{name} && $_->{rpm} ge $rpm->{rpm} } @latest;
156
157			$versions{$rpm->{name}}{$rpm->{version}}++ unless grep { $_->{name} eq $rpm->{name} && $_->{rpm} eq $rpm->{rpm} } @devel;
158			next if $versions{$rpm->{name}}{$rpm->{version}} > $releases \|\| scalar keys %{$versions{$rpm->{name}}} > $releases;
159			print "Possible update " . $rpm->{rpm}->filename . "\n" if grep { $_->{name} eq $rpm->{name} } @latest;
160
161			push @devel, $rpm;
162			unless ( grep { $_->{repo} =~ m{/smedev$} && $_->{filename} eq $rpm->{filename} } @rpms ) {
163			push @{$copy{"$releasedir/$smerel/smedev/$arch/RPMS"}}, $rpm->{rpm}->filename;
164			$changed{"$releasedir/$smerel/smedev"}++;
165			push @rpms, { name => $rpm->{name},
166			filename => $rpm->{filename},
167			version => $rpm->{version},
168			release => $rpm->{release},
169			repo => "$releasedir/$smerel/smedev",
170			rpm => $rpm->{rpm},
171			source => $rpm->{source}
172			};
173			}
174	slords	1.1	}
175
176	slords	1.11	foreach my $rpm ( grep { $_->{repo} =~ m{/smedev$} } @rpms ) {
177			$rpm->{repo} = 'deleted' unless grep { $_->{filename} eq $rpm->{filename} } @devel;
178	slords	1.2	}
179
180	slords	1.18	print "\nFinding source packages...\n";
181	slords	1.11	foreach my $rpm ( grep { $_->{repo} !~ m{^(builds\|deleted)$} } @rpms ) {
182			my $src;
183			my $srpm = $rpm->{rpm}->tag("SOURCERPM");
184
185			my @sources = ();
186			push @sources, ($src = $srpm);
187			push @sources, $src if (($src = $srpm) =~ s/\.el\d\.rf\./\.rf\./);
188			push @sources, $src if (($src = $srpm) =~ s/\.[0-3]\.el\d+\.rf\./\.rf\./);
189			push @sources, $src if (($src = $srpm) =~ s/\.(el\|fc)\d\.at//);
190			push @sources, $src if (($src = $srpm) =~ s/\.at//);
191			push @sources, $src if (($src = $srpm) =~ s/\.(el\|fc)\d//);
192			push @sources, $src if (($src = $srpm) =~ s/\.\d+\.(el\|fc)\d//);
193
194			my $found = 0;
195			my $repo = $rpm->{repo};
196			foreach my $source ( @sources ) {
197			if ( $sources{$source} ) {
198	slords	1.18	if ($repo eq $stagedir) {
199			push @{$copy{"$repo/SRPMS-$arch"}}, "$sources{$source}/$source" unless -f "$repo/SRPMS-$arch/$source";
200			} else {
201			unless ( -f "$repo/SRPMS/$source" ) {
202			push @{$copy{"$repo/SRPMS"}}, "$sources{$source}/$source";
203			$changed{$repo}++;
204			}
205	slords	1.11	}
206			$srpms{$repo}{$source}++;
207			$found++;
208			last;
209			}
210			}
211			print "Can't find source for ".$rpm->{rpm}->filename."\n" unless $found;
212	slords	1.1	}
213
214	slords	1.11	my @remove;
215			foreach my $repo ( keys %srpms ) {
216			foreach my $rpm ( keys %{$srpms{$repo}} ) {
217			unless ( $srpms{$repo}{$rpm} ) {
218	slords	1.18	if ($repo eq $stagedir) {
219			push @remove, "$repo/SRPMS-$arch/$rpm";
220			} else {
221			push @remove, "$repo/SRPMS/$rpm";
222			$changed{$repo}++;
223			}
224	slords	1.11	}
225			}
226	slords	1.1	}
227
228			print "\n";
229			foreach my $repo ( sort keys %copy ) {
230	slords	1.12	foreach my $rpm ( keys %{{ map { $_ => 1 } @{$copy{$repo}} }} ) {
231	slords	1.2	print "copying $rpm to $repo/\n";
232	slords	1.17	mkdir "$repo" unless -d "$repo";
233	slords	1.2	system(qw(cp --preserve=timestamps), $rpm, "$repo/");
234			}
235	slords	1.1	}
236
237			print "\n";
238	slords	1.12	foreach my $rpm ( keys %{{ map { $_ => 1 } (@remove, map { $_->{rpm}->filename } grep { $_->{repo} eq 'deleted' } @rpms) }} ) {
239	slords	1.2	print "removing $rpm\n";
240			unlink $rpm;
241	slords	1.1	}
242
243			print "\n";
244			foreach my $repo ( sort keys %changed ) {
245	slords	1.7	my $dir=`readlink -f $repo/$arch`;
246			chomp $dir;
247			print "rebuilding $dir...\n";
248			system(qw(createrepo), "$dir");
249	slords	1.17	system(qw(rm -rf), "$dir/.olddata") if -d "$dir/.olddata";
250	slords	1.7	system(qw(repoview), "$dir");
251	slords	1.17	system(qw(rm -rf), "$dir/.repodata") if "$dir/.repodata";
252	slords	1.1	}