--- builds_bin/update_repos	2014/10/11 16:43:16	1.78
+++ builds_bin/update_repos	2017/02/14 21:38:34	1.88
@@ -16,6 +16,11 @@ $| = 1;
 my %opts;
 getopts( 'ostr:', \%opts );
 $opts{r} ||= '';
+#available options
+#-o = verify we have originals of the packages in repos (not really used anymore)
+#-s = verify every package has a source package (so we can rebuild if necessary)
+#-t = test run (don't do anything just print out what would be done)
+#-r {regex} = regex of packages to run the script against
 
 my $rpm_flags = RPM2->vsf_nodsaheader | RPM2->vsf_norsaheader | RPM2->vsf_nodsa | RPM2->vsf_norsa;
 
@@ -45,7 +50,7 @@ my $distrepo = { '8' => { active    => 1
                           keyname   => 'SME Server 7 signing key <bugteam@contribs.org>',
                           checksum  => 'sha256',
                         },
-                 '10' => { active   => 0,
+                 '10' => { active   => 1,
                           centos    => 7,
                           archs     => [ 'x86_64' ],
                           repo      => '/build/smeserver/repo/testing/10/',
@@ -53,11 +58,18 @@ my $distrepo = { '8' => { active    => 1
                           community => '/build/builds/smeserver-10-contribs/',
                           stage     => '/build/smeserver/stage/10/',
                           sign      => 1,
-                          keyname   => 'SME Server 7 signing key <bugteam@contribs.org>',
+                          keyname   => 'Koozali SME Server signing key (10) <bugteam@koozali.org>',
                           checksum  => 'sha256',
                         },
                };
 
+# list here repo and priority
+# prio: highest priority will hide lowest priority
+# inc: include for iso ?
+# ver:
+# rel:
+# orig:
+# base, os, stage:
 my $repos = { "${repotag}os"              => { prio => 19, inc => 1, ver => 1, rel => 1, os => 1 },
               "${repotag}updates"         => { prio => 18, inc => 1, ver => 1, rel => 1 },
               "${repotag}updates-testing" => { prio => 17, inc => 1, ver => 1, rel => 3 },
@@ -67,7 +79,9 @@ my $repos = { "${repotag}os"
               "${repotag}test"            => { prio => 13, inc => 0, ver => 2, rel => 3, devel => 2 },
               "${repotag}dev"             => { prio => 12, inc => 0, ver => 1, rel => 1, devel => 1 },
               'centos'                    => { prio => 10, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 },
-              'epel'                      => { prio =>  9, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 },
+              'remi'                      => { prio =>  9, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 },
+              'remisafe'                  => { prio =>  9, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 },
+              'epel'                      => { prio =>  8, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 },
               'rpmforge'                  => { prio =>  7, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 },
               'atrpms'                    => { prio =>  6, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 },
               'builds'                    => { prio =>  4, inc => 0, ver => 0, rel => 0, orig => 1 },
@@ -75,6 +89,9 @@ my $repos = { "${repotag}os"
               'stage'                     => { prio =>  1, inc => 0, ver => 0, rel => 0, stage => 1, os => 1 },
             };
 
+# list here the path on our local mirror to each upstream and our mirrors
+# ~A will be expanded to architecture
+# ~C will be expanded to Centos main version number, i.e. 7 for SME 10. 
 my $baserepo = { "~S${repotag}os/~A/" => "${repotag}os",
                  "~S${repotag}updates/~A/" => "${repotag}updates",
                  "~S${repotag}updates-testing/~A/" => "${repotag}updates-testing",
@@ -94,6 +111,10 @@ my $baserepo = { "~S${repotag}os/~A/" =>
                  '/mirrors/centos-vault/~C/extras/~A/' => 'centos',
                  '/mirrors/fedora/epel/~C/~A/' => 'epel',
                  '/mirrors/fedora/epel/testing/~C/~A/' => 'epel',
+                 '/mirrors/remi/enterprise/~C/remi/~A/' => 'remi',
+                 '/mirrors/remi/SRPMS/' => 'remi',
+                 '/mirrors/remi/enterprise/~C/safe/~A/' => 'remisafe',
+                 '/mirrors/remi/SRPMS/' => 'remisafe',
                  '/mirrors/rpmforge/redhat/el~C/en/~A/dag/' => 'rpmforge',
                  '/mirrors/rpmforge/redhat/el~C/en/~A/extras/' => 'rpmforge',
                  '/mirrors/rpmforge/source/' => 'rpmforge',
@@ -103,19 +124,35 @@ my $baserepo = { "~S${repotag}os/~A/" =>
                  '/mirrors/atrpms/src/el~C-~A/atrpms/testing/' => 'atrpms',
                };
 
+# add here packages to exclude
+# for each version, if a repo is listed and rpm are listed they will be ignored fromt his repo
 my $skippkg = { '8'  => { 'atrpms'        => { map { $_ => 1 } ( 'clamav', 'gnupg' ) },
                           'centos'        => { map { $_ => 1 } ( 'horde', 'imp-h3', 'ingo-h3', 'turba-h3' ) },
-                          'epel'          => { map { $_ => 1 } ( 'clamav', 'fping', 'html2ps', 'horde', 'libtalloc', 'libtdb', 'oidentd', 'perl-Compress-Bzip2', 'perl-Compress-Raw-Zlib', 'perl-Compress-Raw-Bzip2', 'perl-Crypt-OpenSSL-Bignum', 'perl-Crypt-OpenSSL-Random', 'perl-Crypt-OpenSSL-RSA', 'perl-Razor-Agent', 'smolt' ) },
-                          'rpmforge'      => { map { $_ => 1 } ( 'dovecot', 'libtalloc', 'mod_auth_tkt', 'perl-Test-Inline', 'spamassassin' ) },
+                          'epel'          => { map { $_ => 1 } ( 'clamav', 'cppunit', 'fping', 'html2ps', 'horde', 'libtalloc', 'libtdb', 'oidentd', 'perl-Compress-Bzip2', 'perl-Compress-Raw-Zlib', 'perl-Compress-Raw-Bzip2', 'perl-Crypt-OpenSSL-Bignum', 'perl-Crypt-OpenSSL-Random', 'perl-Crypt-OpenSSL-RSA', 'perl-Razor-Agent', 'smolt' ) },
+                          'rpmforge'      => { map { $_ => 1 } ( 'dovecot', 'libtalloc', 'mod_auth_tkt', 'perl-Test-Inline', 'spamassassin', 'lzo',  'perl-Convert-TNEF', 'perl-Mail-SPF', 'proftpd', 'erlang', 'perl-DateTime-Format-Mail', 'perl-DateTime-Format-W3CDTF', 'perl-Email-Abstract', 'perl-Email-MIME-Attachment-Stripper', 'perl-Email-Reply', 'perl-Font-AFM', 'perl-IPC-Run', 'rkhunter') },
                         },
                 '9'  => { 'atrpms'        => { map { $_ => 1 } ( 'clamav', 'perl-Pod-Escapes', 'perl-Pod-Simple' ) },
                           'epel'          => { map { $_ => 1 } ( 'oidentd', 'perl-Razor-Agent' ) },
-                          'rpmforge'      => { map { $_ => 1 } ( 'perl-CGI', 'perl-Compress-Raw-Zlib', 'perl-ExtUtils-ParseXS', 'perl-Pod-Escapes', 'perl-Pod-Simple', 'perl-Time-HiRes', 'perl-version', 'spamassassin' ) },
+                          'rpmforge'      => { map { $_ => 1 } ( 'perl-CGI', 'perl-Compress-Raw-Zlib', 'perl-ExtUtils-ParseXS', 'perl-Pod-Escapes', 'perl-Pod-Simple', 'perl-Time-HiRes', 'perl-version', 'spamassassin', 'portreserve', 'perl-libwww-perl' , 'lzo', 'perl-BSD-Resource', 'perl-Convert-ASN1', 'perl-Devel-StackTrace', 'perl-Digest-SHA1', 'perl-MIME-tools', 'perl-MailTools', 'perl-PPI', 'perl-URI', 'perl-XML-NamespaceSupport', 'perl-XML-Parser', 'perl-XML-SAX-Writer', 'perl-YAML-Syck', 'proftpd', 'syslinux') },
                         },
                 '10' => { 
                         },
               };
 
+# add here include only conditions
+# for each version if a repo is listed here, only listed rpm will be considered
+my $onlypkg = { '8' => {
+                          'remi' => { },
+                          'remisafe' => { },
+			},
+		'9'  => {
+			  'remi' => { },
+			},
+                '10' => {
+			  'remi' => { map { $_ => 1 } ( 'php-pear' )},
+                        },
+};
+
 my ($stage) = sort { $repos->{$a}->{stage} <=> $repos->{$b}->{stage} } grep { $repos->{$_}->{stage} } keys %$repos;
 my ($devel1, $devel2) = sort { $repos->{$a}->{devel} <=> $repos->{$b}->{devel} } grep { $repos->{$_}->{devel} } keys %$repos;
 $devel2 ||= $devel1;
@@ -129,8 +166,8 @@ foreach my $ver ( sort { $a <=> $b } key
     %sources = ();
     %base = ();
 
-    @{$distrepo->{$ver}->{all_archs}} = ('noarch', @{$distrepo->{$ver}->{archs}});
-    if ( grep $_ eq 'i386', @{$distrepo->{$ver}->{archs}} ) {
+    @{$distrepo->{$ver}->{all_archs}} = ('noarch', @{$distrepo->{$ver}->{'archs'}});
+    if ( grep $_ eq 'i386', @{$distrepo->{$ver}->{'archs'}} ) {
         push @{$distrepo->{$ver}->{all_archs}}, ('i586', 'i686');
     }
 
@@ -139,7 +176,7 @@ foreach my $ver ( sort { $a <=> $b } key
         $bdir =~ s/~S/$distrepo->{$ver}->{repo}/;
         $bdir =~ s/~s/$ver/;
         $bdir =~ s/~C/$distrepo->{$ver}->{centos}/;
-        for my $arch ( @{$distrepo->{$ver}->{archs}}, 'SRPMS/' ) {
+        for my $arch ( @{$distrepo->{$ver}->{'archs'}}, 'SRPMS/' ) {
             my $adir = $bdir;
             if ( $arch eq 'SRPMS/' ) {
                 $adir =~ s/~A.*//;
@@ -162,13 +199,21 @@ foreach my $ver ( sort { $a <=> $b } key
             }
             next unless -d $adir;
             opendir DIR, $adir or next;
-            process_rpm("$adir$_", $ver, $baserepo->{$dir}) foreach readdir DIR;
+            my @txt = grep {  /\.rpm$/ && -f "$adir/$_" }  readdir DIR;
             closedir DIR;
+            if (scalar(@txt)) {
+               opendir DIR, $adir or next;
+               process_rpm("$adir$_", $ver, $baserepo->{$dir})  foreach readdir DIR;
+               closedir DIR;
+            } elsif ($baserepo->{$dir} eq 'epel') {
+               # EPEL 7  uses a structure with subfolders of first letter of package
+	       find( { wanted => sub { process_rpm($_, $ver, $baserepo->{$dir}) if ($_ =~ m/\.rpm$/);}, no_chdir => 1, follow_fast => 1 }, $adir );
+            }
             last unless $dir =~ m{~A};
         }
     }
-    find( { wanted => sub { process_rpm($_, $ver, 'builds'); }, no_chdir => 1, follow_fast => 1 }, "$_" ) foreach @{$distrepo->{$ver}->{builds}};
-    find( { wanted => sub { process_rpm($_, $ver, 'community'); }, no_chdir => 1, follow_fast => 1 }, $distrepo->{$ver}->{community} ) if $distrepo->{$ver}->{community};
+    find( { wanted => sub { process_rpm($_, $ver, 'builds'); }, no_chdir => 1, follow_fast => 1 }, "$_" ) foreach @{$distrepo->{$ver}->{'builds'}};
+    find( { wanted => sub { process_rpm($_, $ver, 'community'); }, no_chdir => 1, follow_fast => 1 }, $distrepo->{$ver}->{'community'} ) if $distrepo->{$ver}->{'community'};
 
     foreach my $base ( sort keys %$rpms ) {
         next unless $rpms->{$base}->{rpms};
@@ -394,7 +439,7 @@ foreach my $ver ( sort { $a <=> $b } key
                 foreach my $a ( keys %{$acnt->{$r}->{$p}} ) {
                     if ( $a eq 'noarch' ) {
                         my $p2 = ${$acnt->{$r}->{$p}->{$a}}[0];
-                        if ( ! $p2->{repos} && scalar @{$acnt->{$r}->{$p}->{$a}} != scalar @{$distrepo->{$ver}->{archs}} ) {
+                        if ( ! $p2->{repos} && scalar @{$acnt->{$r}->{$p}->{$a}} != scalar @{$distrepo->{$ver}->{'archs'}} ) {
                             $p2->{oldrepo} = $p2->{repo};
                             push @{$p2->{repos}}, $p2->{repo};
                         }
@@ -452,7 +497,10 @@ foreach my $ver ( sort { $a <=> $b } key
                     if ( $pkg->{src} ) {
                         push @d, "${head}SRPMS/" . basename($pkg->{rpm}->filename);
                     } elsif ( $pkg->{rpm}->arch eq 'noarch' ) {
-                        push @d, map { "${head}$_/$tail" . basename($pkg->{rpm}->filename) } @{$distrepo->{$ver}->{archs}};
+                        push @d, map { "${head}$_/$tail" . basename($pkg->{rpm}->filename) } @{$distrepo->{$ver}->{'archs'}};
+                    } elsif ($ver>=10 && $pkg->{rpm}->arch =~ m{^(i[356]86)$}  && $pkg->{arch} =~ m{^(x86_64)$}  ) {
+                       #add copy of i686 packages in x86_64 folder for SME>=10 
+                       push @d, map { "${head}$_/$tail" . basename($pkg->{rpm}->filename) } @{$distrepo->{$ver}->{'archs'}};
                     } elsif ( $pkg->{rpm}->arch =~ m{^(i[356]86)$} ) {
                         push @d, "${head}i386/$tail" . basename($pkg->{rpm}->filename);
                     } else {
@@ -496,7 +544,7 @@ foreach my $ver ( sort { $a <=> $b } key
     }
     foreach my $repo ( sort { $repos->{$b}->{prio} <=> $repos->{$a}->{prio} } keys %repochg ) {
         next if $repos->{$repo}->{orig} || $repos->{$repo}->{stage};
-        foreach my $arch ( @{$distrepo->{$ver}->{archs}} ) {
+        foreach my $arch ( @{$distrepo->{$ver}->{'archs'}} ) {
             my $dir = $distrepo->{$ver}->{repo} . "$repo/$arch";
             $dir = qx(readlink -f $dir);
             if ( $dir ) {
@@ -527,11 +575,11 @@ foreach my $ver ( sort { $a <=> $b } key
         }
     }
     unless ( $opts{t} ) {
-        foreach (@{$distrepo->{$ver}->{builds}})
+        foreach (@{$distrepo->{$ver}->{'builds'}})
         {
             finddepth(sub{rmdir}, "$_") if -d "$_";
         }
-        finddepth(sub{rmdir}, $distrepo->{$ver}->{community}) if -d "$distrepo->{$ver}->{community}";
+        finddepth(sub{rmdir}, $distrepo->{$ver}->{'community'}) if -d "$distrepo->{$ver}->{'community'}";
     }
 }
 
@@ -593,6 +641,7 @@ sub process_rpm {
     }
     my ($src, $version, $release) = ($pkg->is_source_package ? $pkg->filename : $pkg->sourcerpm) =~ m{(?:^|/)([^/]*)-([^-]+)-([^-]+)\.\w+\.rpm$};
     return if $skippkg->{$ver}->{$repo} && ( $skippkg->{$ver}->{$repo}->{$src} || $skippkg->{$ver}->{$repo}->{$pkg->name} );
+    return if $onlypkg->{$ver}->{$repo} && !( $onlypkg->{$ver}->{$repo}->{$src} || $onlypkg->{$ver}->{$repo}->{$pkg->name} );
     return if $repos->{$repo}->{base} && $nosig && $baserepo->{$ver}->{sign};
     if ( $nosig && ! -f "$HOME/.rpmpass" ) { print " * missing signature (" .$pkg->filename . ")\n"; return; }
     my $arch = 'unknown';