--- builds_bin/update_repos 2016/03/05 15:57:15 1.79 +++ builds_bin/update_repos 2024/04/28 18:53:35 1.94 @@ -16,14 +16,19 @@ $| = 1; my %opts; getopts( 'ostr:', \%opts ); $opts{r} ||= ''; +#available options +#-o = verify we have originals of the packages in repos (not really used anymore) +#-s = verify every package has a source package (so we can rebuild if necessary) +#-t = test run (don't do anything just print out what would be done) +#-r {regex} = regex of packages to run the script against my $rpm_flags = RPM2->vsf_nodsaheader | RPM2->vsf_norsaheader | RPM2->vsf_nodsa | RPM2->vsf_norsa; -my $osdir = 'SME'; +my $osdir = 'Packages'; my $disttag = 'sme'; my $repotag = 'sme'; -my $distrepo = { '8' => { active => 1, +my $distrepo = { '8' => { active => 0, centos => 5, archs => [ 'i386', 'x86_64' ], repo => '/build/smeserver/repo/8/', @@ -34,7 +39,7 @@ my $distrepo = { '8' => { active => 1 keyname => 'SME Server 7 signing key ', checksum => 'sha', }, - '9' => { active => 1, + '9' => { active => 0, centos => 6, archs => [ 'i386', 'x86_64' ], repo => '/build/smeserver/repo/9/', @@ -45,36 +50,53 @@ my $distrepo = { '8' => { active => 1 keyname => 'SME Server 7 signing key ', checksum => 'sha256', }, - '10' => { active => 0, + '10' => { active => 1, centos => 7, archs => [ 'x86_64' ], - repo => '/build/smeserver/repo/testing/10/', + repo => '/build/smeserver/repo/10/', builds => ['/build/builds/smeserver-10-core/'], community => '/build/builds/smeserver-10-contribs/', stage => '/build/smeserver/stage/10/', sign => 1, - keyname => 'SME Server 7 signing key ', + keyname => 'Koozali SME Server signing key (10) ', checksum => 'sha256', }, }; -my $repos = { "${repotag}os" => { prio => 19, inc => 1, ver => 1, rel => 1, os => 1 }, - "${repotag}updates" => { prio => 18, inc => 1, ver => 1, rel => 1 }, - "${repotag}updates-testing" => { prio => 17, inc => 1, ver => 1, rel => 3 }, - "${repotag}extras" => { prio => 16, inc => 0, ver => 1, rel => 1 }, - "${repotag}addons" => { prio => 15, inc => 0, ver => 1, rel => 1 }, - "${repotag}contribs" => { prio => 14, inc => 0, ver => 1, rel => 1 }, - "${repotag}test" => { prio => 13, inc => 0, ver => 2, rel => 3, devel => 2 }, - "${repotag}dev" => { prio => 12, inc => 0, ver => 1, rel => 1, devel => 1 }, - 'centos' => { prio => 10, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 }, - 'epel' => { prio => 9, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 }, - 'rpmforge' => { prio => 7, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 }, - 'atrpms' => { prio => 6, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 }, +# list here repo and priority +# prio: highest priority will hide lowest priority +# inc: include for iso ? +# ver: +# rel: +# orig: +# base, os, stage: +my $repos = { "${repotag}os" => { prio => 29, inc => 1, ver => 1, rel => 1, os => 1 }, + "${repotag}updates" => { prio => 28, inc => 1, ver => 1, rel => 1 }, + "${repotag}updates-testing" => { prio => 27, inc => 1, ver => 1, rel => 3 }, + "${repotag}extras" => { prio => 26, inc => 0, ver => 1, rel => 1 }, + "${repotag}addons" => { prio => 25, inc => 0, ver => 1, rel => 1 }, + "${repotag}contribs" => { prio => 24, inc => 0, ver => 1, rel => 1 }, + "${repotag}test" => { prio => 23, inc => 0, ver => 2, rel => 3, devel => 2 }, + "${repotag}dev" => { prio => 22, inc => 0, ver => 1, rel => 1, devel => 1 }, + 'centos' => { prio => 21, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 }, + 'remisafe' => { prio => 12, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 }, + 'remi' => { prio => 11, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 }, + 'epel' => { prio => 10, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 }, + 'epeltesting' => { prio => 9, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 }, + 'rpmfusion' => { prio => 8, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 }, + 'openfusion' => { prio => 7, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 }, + 'rpmforge' => { prio => 6, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 }, + 'atrpms' => { prio => 5, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 }, 'builds' => { prio => 4, inc => 0, ver => 0, rel => 0, orig => 1 }, 'community' => { prio => 2, inc => 0, ver => 0, rel => 0, orig => 1 }, 'stage' => { prio => 1, inc => 0, ver => 0, rel => 0, stage => 1, os => 1 }, }; +# list here the path on our local mirror to each upstream and our mirrors +# ~A will be expanded to architecture +# ~C will be expanded to Centos main version number, i.e. 7 for SME 10. +# ~s will be expanded to SME version +# ~S expanded to repo root my $baserepo = { "~S${repotag}os/~A/" => "${repotag}os", "~S${repotag}updates/~A/" => "${repotag}updates", "~S${repotag}updates-testing/~A/" => "${repotag}updates-testing", @@ -93,7 +115,11 @@ my $baserepo = { "~S${repotag}os/~A/" => '/mirrors/centos-vault/~C/os/~A/' => 'centos', '/mirrors/centos-vault/~C/extras/~A/' => 'centos', '/mirrors/fedora/epel/~C/~A/' => 'epel', - '/mirrors/fedora/epel/testing/~C/~A/' => 'epel', + '/mirrors/fedora/epel/testing/~C/~A/' => 'epeltesting', + '/mirrors/remi/enterprise/~C/remi/~A/' => 'remi', + '/mirrors/remi/SRPMS/' => 'remi', + '/mirrors/remi/enterprise/~C/safe/~A/' => 'remisafe', + '/mirrors/rpmfusion/updates/~C/~A/' => 'rpmfusion', '/mirrors/rpmforge/redhat/el~C/en/~A/dag/' => 'rpmforge', '/mirrors/rpmforge/redhat/el~C/en/~A/extras/' => 'rpmforge', '/mirrors/rpmforge/source/' => 'rpmforge', @@ -101,21 +127,47 @@ my $baserepo = { "~S${repotag}os/~A/" => '/mirrors/atrpms/el~C-~A/atrpms/testing/' => 'atrpms', '/mirrors/atrpms/src/el~C-~A/atrpms/stable/' => 'atrpms', '/mirrors/atrpms/src/el~C-~A/atrpms/testing/' => 'atrpms', + '/mirrors/openfusion/centos~C-~A/' => 'openfusion', + '/mirrors/openfusion/srpms/' => 'openfusion', }; +# add here packages to exclude +# for each version, if a repo is listed and rpm are listed they will be ignored fromt his repo my $skippkg = { '8' => { 'atrpms' => { map { $_ => 1 } ( 'clamav', 'gnupg' ) }, 'centos' => { map { $_ => 1 } ( 'horde', 'imp-h3', 'ingo-h3', 'turba-h3' ) }, - 'epel' => { map { $_ => 1 } ( 'clamav', 'fping', 'html2ps', 'horde', 'libtalloc', 'libtdb', 'oidentd', 'perl-Compress-Bzip2', 'perl-Compress-Raw-Zlib', 'perl-Compress-Raw-Bzip2', 'perl-Crypt-OpenSSL-Bignum', 'perl-Crypt-OpenSSL-Random', 'perl-Crypt-OpenSSL-RSA', 'perl-Razor-Agent', 'smolt' ) }, - 'rpmforge' => { map { $_ => 1 } ( 'dovecot', 'libtalloc', 'mod_auth_tkt', 'perl-Test-Inline', 'spamassassin' ) }, + 'epel' => { map { $_ => 1 } ( 'clamav', 'cppunit', 'fping', 'html2ps', 'horde', 'libtalloc', 'libtdb', 'oidentd', 'perl-Compress-Bzip2', 'perl-Compress-Raw-Zlib', 'perl-Compress-Raw-Bzip2', 'perl-Crypt-OpenSSL-Bignum', 'perl-Crypt-OpenSSL-Random', 'perl-Crypt-OpenSSL-RSA', 'perl-Razor-Agent', 'smolt' ) }, + 'epeltesting' => { map { $_ => 1 } ( 'clamav', 'cppunit', 'fping', 'html2ps', 'horde', 'libtalloc', 'libtdb', 'oidentd', 'perl-Compress-Bzip2', 'perl-Compress-Raw-Zlib', 'perl-Compress-Raw-Bzip2', 'perl-Crypt-OpenSSL-Bignum', 'perl-Crypt-OpenSSL-Random', 'perl-Crypt-OpenSSL-RSA', 'perl-Razor-Agent', 'smolt' ) }, + 'rpmforge' => { map { $_ => 1 } ( 'dovecot', 'libtalloc', 'mod_auth_tkt', 'perl-Test-Inline', 'spamassassin', 'lzo', 'perl-Convert-TNEF', 'perl-Mail-SPF', 'proftpd', 'erlang', 'perl-DateTime-Format-Mail', 'perl-DateTime-Format-W3CDTF', 'perl-Email-Abstract', 'perl-Email-MIME-Attachment-Stripper', 'perl-Email-Reply', 'perl-Font-AFM', 'perl-IPC-Run', 'rkhunter') }, }, - '9' => { 'atrpms' => { map { $_ => 1 } ( 'clamav', 'perl-Pod-Escapes', 'perl-Pod-Simple' ) }, - 'epel' => { map { $_ => 1 } ( 'oidentd', 'perl-Razor-Agent' ) }, - 'rpmforge' => { map { $_ => 1 } ( 'perl-CGI', 'perl-Compress-Raw-Zlib', 'perl-ExtUtils-ParseXS', 'perl-Pod-Escapes', 'perl-Pod-Simple', 'perl-Time-HiRes', 'perl-version', 'spamassassin' ) }, + '9' => { 'atrpms' => { map { $_ => 1 } ( 'clamav', 'perl-Pod-Escapes', 'perl-Pod-Simple','libvorbis' , 'libogg','iksemel' ) }, + 'epel' => { map { $_ => 1 } ( 'clamav', 'oidentd', 'perl-Razor-Agent' ) }, + 'epeltesting' => { map { $_ => 1 } ( 'clamav', 'oidentd', 'perl-Razor-Agent' ) }, + 'rpmforge' => { map { $_ => 1 } ( 'perl-CGI', 'perl-Compress-Raw-Zlib', 'perl-ExtUtils-ParseXS', 'perl-Pod-Escapes', 'perl-Pod-Simple', 'perl-Time-HiRes', 'perl-version', 'spamassassin', 'portreserve', 'perl-libwww-perl' , 'lzo', 'perl-BSD-Resource', 'perl-Convert-ASN1', 'perl-DBD-SQLite', 'perl-Devel-StackTrace', 'perl-Digest-SHA1', 'perl-JSON', 'perl-MIME-tools', 'perl-MailTools', 'perl-PPI', 'perl-URI', 'perl-XML-NamespaceSupport', 'perl-XML-Parser', 'perl-XML-SAX-Writer', 'perl-YAML-Syck', 'proftpd', 'syslinux') }, }, - '10' => { + '10' => { + #'epel' => { map { $_ => 1 } ( 'clamav') }, + 'epeltesting' => { map { $_ => 1 } ( 'clamav', 'tidy') }, + 'openfusion' => { map { $_ => 1 } ( 'daemontools-encore' , 'daemontool', 'perl-Apache-AuthTkt', 'perl-Data-Validate-Domain', 'perl-HTML-Tabulate', 'perl-IP-Country', 'perl-Object-Accessor','perl-Archive-Tar', 'perl-Authen-SASL', 'perl-Business-ISBN', 'perl-Business-ISBN-Data', 'perl-Config-Tiny', 'perl-Convert-ASN1', 'perl-DBD-MySQL', 'perl-DB_File', 'perl-Data-Dumper', 'perl-Date-Manip', 'perl-Email-Address', 'perl-Encode', 'perl-ExtUtils-MakeMaker', 'perl-ExtUtils-Manifest', 'perl-ExtUtils-ParseXS', 'perl-File-ShareDir', 'perl-Getopt-Long', 'perl-HTML-Parser', 'perl-HTTP-Message', 'perl-IO-HTML', 'perl-IO-Socket-INET6', 'perl-MailTools', 'perl-Mozilla-CA', 'perl-Net-SSLeay', 'perl-NetAddr-IP', 'perl-PathTools', 'perl-Readonly', 'perl-Storable', 'perl-Test-Simple', 'perl-Text-Unidecode', 'perl-XML-Parser', 'perl-libwww-perl', 'perl-parent' ) }, }, }; +# add here include only conditions +# for each version if a repo is listed here, only listed rpm will be considered +my $onlypkg = { '8' => { + 'openfusion' => { }, + 'remi' => { }, + 'remisafe' => { }, + }, + '9' => { + 'openfusion' => { map { $_ => 1 } ( 'perl-Carp', 'perl-Scalar-List-Utils', 'perl-Class-Method-Modifiers', 'perl-B-Hooks-OP-Check', 'perl-Sub-Exporter-Progressive', 'perl-Devel-GlobalDestruction', 'perl-Socket', 'perl-Data-Validate-IP', 'perl-Eval-Closure', 'perl-Params-Classify', 'perl-Module-Runtime', 'perl-Module-Implementation', 'perl-Class-Load-XS', 'perl-MooX-Types-MooseLike', 'perl-Hash-FieldHash', 'perl-Algorithm-C3', 'perl-Class-C3', 'perl-MRO-Compat', 'perl-Class-XSAccessor', 'perl-ExtUtils-Manifest', 'perl-IPC-Cmd', 'perl-Module-Build', 'perl-ExtUtils-CBuilder', 'perl-Perl-OSType', 'perl-ExtUtils-Install', 'perl-Params-Validate', 'perl-Sub-Name', 'perl-Sub-Quote', 'perl-Sub-Identify', 'perl-Role-Tiny', 'perl-File-HomeDir', 'perl-Data-Dumper-Concise', 'perl-multidimensional', 'perl-Test-Warnings', 'perl-Term-ANSIColor', 'perl-Task-Weaken', 'perl-Clone-PP', 'perl-Data-IEEE754', 'perl-DateTime-TimeZone', 'perl-Exporter-Tiny', 'perl-List-AllUtils', 'perl-Dist-CheckConflicts', 'perl-Moose', 'perl-DateTime-Locale', 'perl-DateTime', 'perl-Variable-Magic', 'perl-B-Hooks-EndOfScope', 'perl-namespace-clean', 'perl-namespace-autoclean', 'perl-Lexical-SealRequireHints', 'perl-bareword-filehandles', 'perl-indirect', 'perl-strictures', 'perl-Moo', 'perl-MooX-StrictConstructor', 'perl-Throwable', 'perl-Math-Int64', 'perl-Math-Int128', 'perl-MaxMind-DB-Common', 'perl-Net-Works', 'perl-Sort-Naturally', 'perl-Data-Printer', 'perl-MaxMind-DB-Reader', 'perl-GeoIP2', 'perl-Data-OptList','perl-Class-Load', 'perl-App-cpanminus' )}, # not for moment : perl-Devel-StackTrace perl-parent perl-ExtUtils-MakeMaker perl-List-MoreUtils + 'remi' => { }, + }, + '10' => { + 'remi' => { map { $_ => 1 } ( 'php-pear' )}, + }, +}; + + my ($stage) = sort { $repos->{$a}->{stage} <=> $repos->{$b}->{stage} } grep { $repos->{$_}->{stage} } keys %$repos; my ($devel1, $devel2) = sort { $repos->{$a}->{devel} <=> $repos->{$b}->{devel} } grep { $repos->{$_}->{devel} } keys %$repos; $devel2 ||= $devel1; @@ -135,7 +187,7 @@ foreach my $ver ( sort { $a <=> $b } key } foreach my $dir ( sort { $repos->{$baserepo->{$b}}->{prio} <=> $repos->{$baserepo->{$a}}->{prio} || $a cmp $b } keys %$baserepo ) { - my $bdir = $dir; + my $bdir = $dir; $bdir =~ s/~S/$distrepo->{$ver}->{repo}/; $bdir =~ s/~s/$ver/; $bdir =~ s/~C/$distrepo->{$ver}->{centos}/; @@ -162,8 +214,16 @@ foreach my $ver ( sort { $a <=> $b } key } next unless -d $adir; opendir DIR, $adir or next; - process_rpm("$adir$_", $ver, $baserepo->{$dir}) foreach readdir DIR; + my @txt = grep { /\.rpm$/ && -f "$adir/$_" } readdir DIR; closedir DIR; + if (scalar(@txt)) { + opendir DIR, $adir or next; + process_rpm("$adir$_", $ver, $baserepo->{$dir}) foreach readdir DIR; + closedir DIR; + } elsif ($baserepo->{$dir} eq 'epel') { + # EPEL 7 uses a structure with subfolders of first letter of package + find( { wanted => sub { process_rpm($_, $ver, $baserepo->{$dir}) if ($_ =~ m/\.rpm$/);}, no_chdir => 1, follow_fast => 1 }, $adir ); + } last unless $dir =~ m{~A}; } } @@ -399,7 +459,7 @@ foreach my $ver ( sort { $a <=> $b } key push @{$p2->{repos}}, $p2->{repo}; } } elsif ( scalar @{$acnt->{$r}->{$p}->{$a}} != 1 ) { - print "\n * many packages ($r, $a, $p)\n" unless $p =~ m{^syslinux-nonlinux-[0-9]}; + print "\n * many packages ($r, $a, $p)\n" unless ( ($p =~ m{^syslinux-nonlinux-[0-9]}) || ($p =~ m{^lib(mcrypt|webp)-[0-9]}) ) ; } } } @@ -453,6 +513,9 @@ foreach my $ver ( sort { $a <=> $b } key push @d, "${head}SRPMS/" . basename($pkg->{rpm}->filename); } elsif ( $pkg->{rpm}->arch eq 'noarch' ) { push @d, map { "${head}$_/$tail" . basename($pkg->{rpm}->filename) } @{$distrepo->{$ver}->{'archs'}}; + } elsif ($ver>=10 && $pkg->{rpm}->arch =~ m{^(i[356]86)$} && $pkg->{arch} =~ m{^(x86_64)$} ) { + #add copy of i686 packages in x86_64 folder for SME>=10 + push @d, map { "${head}$_/$tail" . basename($pkg->{rpm}->filename) } @{$distrepo->{$ver}->{'archs'}}; } elsif ( $pkg->{rpm}->arch =~ m{^(i[356]86)$} ) { push @d, "${head}i386/$tail" . basename($pkg->{rpm}->filename); } else { @@ -509,7 +572,7 @@ foreach my $ver ( sort { $a <=> $b } key my $checksum = "--checksum $distrepo->{$ver}->{checksum}"; my $exclude = ""; - $exclude .= " --exclude ${osdir}/*" if -d "$dir/Packages"; + $exclude .= " --exclude ${osdir}/*" if ( -d "$dir/Packages" && ${osdir} ne "Packages" ); $exclude .= " --exclude *smeserver-release*.rpm" if $dir =~ m{test/|-testing/}; my $groupfile = ""; @@ -593,6 +656,7 @@ sub process_rpm { } my ($src, $version, $release) = ($pkg->is_source_package ? $pkg->filename : $pkg->sourcerpm) =~ m{(?:^|/)([^/]*)-([^-]+)-([^-]+)\.\w+\.rpm$}; return if $skippkg->{$ver}->{$repo} && ( $skippkg->{$ver}->{$repo}->{$src} || $skippkg->{$ver}->{$repo}->{$pkg->name} ); + return if $onlypkg->{$ver}->{$repo} && !( $onlypkg->{$ver}->{$repo}->{$src} || $onlypkg->{$ver}->{$repo}->{$pkg->name} ); return if $repos->{$repo}->{base} && $nosig && $baserepo->{$ver}->{sign}; if ( $nosig && ! -f "$HOME/.rpmpass" ) { print " * missing signature (" .$pkg->filename . ")\n"; return; } my $arch = 'unknown';