/[smeserver]/builds_bin/update_repos
ViewVC logotype

Diff of /builds_bin/update_repos

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph | View Patch Patch

Revision 1.78 by slords, Sat Oct 11 16:43:16 2014 UTC Revision 1.89 by unnilennium, Sat Apr 8 20:27:18 2017 UTC
# Line 16  $| = 1; Line 16  $| = 1;
16  my %opts;  my %opts;
17  getopts( 'ostr:', \%opts );  getopts( 'ostr:', \%opts );
18  $opts{r} ||= '';  $opts{r} ||= '';
19    #available options
20    #-o = verify we have originals of the packages in repos (not really used anymore)
21    #-s = verify every package has a source package (so we can rebuild if necessary)
22    #-t = test run (don't do anything just print out what would be done)
23    #-r {regex} = regex of packages to run the script against
24    
25  my $rpm_flags = RPM2->vsf_nodsaheader | RPM2->vsf_norsaheader | RPM2->vsf_nodsa | RPM2->vsf_norsa;  my $rpm_flags = RPM2->vsf_nodsaheader | RPM2->vsf_norsaheader | RPM2->vsf_nodsa | RPM2->vsf_norsa;
26    
# Line 23  my $osdir = 'SME'; Line 28  my $osdir = 'SME';
28  my $disttag = 'sme';  my $disttag = 'sme';
29  my $repotag = 'sme';  my $repotag = 'sme';
30    
31  my $distrepo = { '8' => { active    => 1,  my $distrepo = { '8' => { active    => 0,
32                            centos    => 5,                            centos    => 5,
33                            archs     => [ 'i386', 'x86_64' ],                            archs     => [ 'i386', 'x86_64' ],
34                            repo      => '/build/smeserver/repo/8/',                            repo      => '/build/smeserver/repo/8/',
# Line 45  my $distrepo = { '8' => { active    => 1 Line 50  my $distrepo = { '8' => { active    => 1
50                            keyname   => 'SME Server 7 signing key <bugteam@contribs.org>',                            keyname   => 'SME Server 7 signing key <bugteam@contribs.org>',
51                            checksum  => 'sha256',                            checksum  => 'sha256',
52                          },                          },
53                   '10' => { active   => 0,                   '10' => { active   => 1,
54                            centos    => 7,                            centos    => 7,
55                            archs     => [ 'x86_64' ],                            archs     => [ 'x86_64' ],
56                            repo      => '/build/smeserver/repo/testing/10/',                            repo      => '/build/smeserver/repo/testing/10/',
# Line 53  my $distrepo = { '8' => { active    => 1 Line 58  my $distrepo = { '8' => { active    => 1
58                            community => '/build/builds/smeserver-10-contribs/',                            community => '/build/builds/smeserver-10-contribs/',
59                            stage     => '/build/smeserver/stage/10/',                            stage     => '/build/smeserver/stage/10/',
60                            sign      => 1,                            sign      => 1,
61                            keyname   => 'SME Server 7 signing key <bugteam@contribs.org>',                            keyname   => 'Koozali SME Server signing key (10) <bugteam@koozali.org>',
62                            checksum  => 'sha256',                            checksum  => 'sha256',
63                          },                          },
64                 };                 };
65    
66    # list here repo and priority
67    # prio: highest priority will hide lowest priority
68    # inc: include for iso ?
69    # ver:
70    # rel:
71    # orig:
72    # base, os, stage:
73  my $repos = { "${repotag}os"              => { prio => 19, inc => 1, ver => 1, rel => 1, os => 1 },  my $repos = { "${repotag}os"              => { prio => 19, inc => 1, ver => 1, rel => 1, os => 1 },
74                "${repotag}updates"         => { prio => 18, inc => 1, ver => 1, rel => 1 },                "${repotag}updates"         => { prio => 18, inc => 1, ver => 1, rel => 1 },
75                "${repotag}updates-testing" => { prio => 17, inc => 1, ver => 1, rel => 3 },                "${repotag}updates-testing" => { prio => 17, inc => 1, ver => 1, rel => 3 },
# Line 66  my $repos = { "${repotag}os" Line 78  my $repos = { "${repotag}os"
78                "${repotag}contribs"        => { prio => 14, inc => 0, ver => 1, rel => 1 },                "${repotag}contribs"        => { prio => 14, inc => 0, ver => 1, rel => 1 },
79                "${repotag}test"            => { prio => 13, inc => 0, ver => 2, rel => 3, devel => 2 },                "${repotag}test"            => { prio => 13, inc => 0, ver => 2, rel => 3, devel => 2 },
80                "${repotag}dev"             => { prio => 12, inc => 0, ver => 1, rel => 1, devel => 1 },                "${repotag}dev"             => { prio => 12, inc => 0, ver => 1, rel => 1, devel => 1 },
81                'centos'                    => { prio => 10, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 },                'centos'                    => { prio => 11, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 },
82                'epel'                      => { prio =>  9, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 },                'remi'                      => { prio => 10, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 },
83                  'remisafe'                  => { prio =>  9, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 },
84                  'epel'                      => { prio =>  8, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 },
85                'rpmforge'                  => { prio =>  7, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 },                'rpmforge'                  => { prio =>  7, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 },
86                'atrpms'                    => { prio =>  6, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 },                'atrpms'                    => { prio =>  6, inc => 0, ver => 0, rel => 0, orig => 1, base => 1 },
87                'builds'                    => { prio =>  4, inc => 0, ver => 0, rel => 0, orig => 1 },                'builds'                    => { prio =>  4, inc => 0, ver => 0, rel => 0, orig => 1 },
# Line 75  my $repos = { "${repotag}os" Line 89  my $repos = { "${repotag}os"
89                'stage'                     => { prio =>  1, inc => 0, ver => 0, rel => 0, stage => 1, os => 1 },                'stage'                     => { prio =>  1, inc => 0, ver => 0, rel => 0, stage => 1, os => 1 },
90              };              };
91    
92    # list here the path on our local mirror to each upstream and our mirrors
93    # ~A will be expanded to architecture
94    # ~C will be expanded to Centos main version number, i.e. 7 for SME 10.
95  my $baserepo = { "~S${repotag}os/~A/" => "${repotag}os",  my $baserepo = { "~S${repotag}os/~A/" => "${repotag}os",
96                   "~S${repotag}updates/~A/" => "${repotag}updates",                   "~S${repotag}updates/~A/" => "${repotag}updates",
97                   "~S${repotag}updates-testing/~A/" => "${repotag}updates-testing",                   "~S${repotag}updates-testing/~A/" => "${repotag}updates-testing",
# Line 94  my $baserepo = { "~S${repotag}os/~A/" => Line 111  my $baserepo = { "~S${repotag}os/~A/" =>
111                   '/mirrors/centos-vault/~C/extras/~A/' => 'centos',                   '/mirrors/centos-vault/~C/extras/~A/' => 'centos',
112                   '/mirrors/fedora/epel/~C/~A/' => 'epel',                   '/mirrors/fedora/epel/~C/~A/' => 'epel',
113                   '/mirrors/fedora/epel/testing/~C/~A/' => 'epel',                   '/mirrors/fedora/epel/testing/~C/~A/' => 'epel',
114                     '/mirrors/remi/enterprise/~C/remi/~A/' => 'remi',
115                     '/mirrors/remi/SRPMS/' => 'remi',
116                     '/mirrors/remi/enterprise/~C/safe/~A/' => 'remisafe',
117    #                 '/mirrors/remi/SRPMS/' => 'remisafe',
118                   '/mirrors/rpmforge/redhat/el~C/en/~A/dag/' => 'rpmforge',                   '/mirrors/rpmforge/redhat/el~C/en/~A/dag/' => 'rpmforge',
119                   '/mirrors/rpmforge/redhat/el~C/en/~A/extras/' => 'rpmforge',                   '/mirrors/rpmforge/redhat/el~C/en/~A/extras/' => 'rpmforge',
120                   '/mirrors/rpmforge/source/' => 'rpmforge',                   '/mirrors/rpmforge/source/' => 'rpmforge',
# Line 103  my $baserepo = { "~S${repotag}os/~A/" => Line 124  my $baserepo = { "~S${repotag}os/~A/" =>
124                   '/mirrors/atrpms/src/el~C-~A/atrpms/testing/' => 'atrpms',                   '/mirrors/atrpms/src/el~C-~A/atrpms/testing/' => 'atrpms',
125                 };                 };
126    
127    # add here packages to exclude
128    # for each version, if a repo is listed and rpm are listed they will be ignored fromt his repo
129  my $skippkg = { '8'  => { 'atrpms'        => { map { $_ => 1 } ( 'clamav', 'gnupg' ) },  my $skippkg = { '8'  => { 'atrpms'        => { map { $_ => 1 } ( 'clamav', 'gnupg' ) },
130                            'centos'        => { map { $_ => 1 } ( 'horde', 'imp-h3', 'ingo-h3', 'turba-h3' ) },                            'centos'        => { map { $_ => 1 } ( 'horde', 'imp-h3', 'ingo-h3', 'turba-h3' ) },
131                            'epel'          => { map { $_ => 1 } ( 'clamav', 'fping', 'html2ps', 'horde', 'libtalloc', 'libtdb', 'oidentd', 'perl-Compress-Bzip2', 'perl-Compress-Raw-Zlib', 'perl-Compress-Raw-Bzip2', 'perl-Crypt-OpenSSL-Bignum', 'perl-Crypt-OpenSSL-Random', 'perl-Crypt-OpenSSL-RSA', 'perl-Razor-Agent', 'smolt' ) },                            'epel'          => { map { $_ => 1 } ( 'clamav', 'cppunit', 'fping', 'html2ps', 'horde', 'libtalloc', 'libtdb', 'oidentd', 'perl-Compress-Bzip2', 'perl-Compress-Raw-Zlib', 'perl-Compress-Raw-Bzip2', 'perl-Crypt-OpenSSL-Bignum', 'perl-Crypt-OpenSSL-Random', 'perl-Crypt-OpenSSL-RSA', 'perl-Razor-Agent', 'smolt' ) },
132                            'rpmforge'      => { map { $_ => 1 } ( 'dovecot', 'libtalloc', 'mod_auth_tkt', 'perl-Test-Inline', 'spamassassin' ) },                            'rpmforge'      => { map { $_ => 1 } ( 'dovecot', 'libtalloc', 'mod_auth_tkt', 'perl-Test-Inline', 'spamassassin', 'lzo',  'perl-Convert-TNEF', 'perl-Mail-SPF', 'proftpd', 'erlang', 'perl-DateTime-Format-Mail', 'perl-DateTime-Format-W3CDTF', 'perl-Email-Abstract', 'perl-Email-MIME-Attachment-Stripper', 'perl-Email-Reply', 'perl-Font-AFM', 'perl-IPC-Run', 'rkhunter') },
133                          },                          },
134                  '9'  => { 'atrpms'        => { map { $_ => 1 } ( 'clamav', 'perl-Pod-Escapes', 'perl-Pod-Simple' ) },                  '9'  => { 'atrpms'        => { map { $_ => 1 } ( 'clamav', 'perl-Pod-Escapes', 'perl-Pod-Simple','libvorbis' , 'libogg' ) },
135                            'epel'          => { map { $_ => 1 } ( 'oidentd', 'perl-Razor-Agent' ) },                            'epel'          => { map { $_ => 1 } ( 'oidentd', 'perl-Razor-Agent' ) },
136                            'rpmforge'      => { map { $_ => 1 } ( 'perl-CGI', 'perl-Compress-Raw-Zlib', 'perl-ExtUtils-ParseXS', 'perl-Pod-Escapes', 'perl-Pod-Simple', 'perl-Time-HiRes', 'perl-version', 'spamassassin' ) },                            'rpmforge'      => { map { $_ => 1 } ( 'perl-CGI', 'perl-Compress-Raw-Zlib', 'perl-ExtUtils-ParseXS', 'perl-Pod-Escapes', 'perl-Pod-Simple', 'perl-Time-HiRes', 'perl-version', 'spamassassin', 'portreserve', 'perl-libwww-perl' , 'lzo', 'perl-BSD-Resource', 'perl-Convert-ASN1', 'perl-Devel-StackTrace', 'perl-Digest-SHA1', 'perl-MIME-tools', 'perl-MailTools', 'perl-PPI', 'perl-URI', 'perl-XML-NamespaceSupport', 'perl-XML-Parser', 'perl-XML-SAX-Writer', 'perl-YAML-Syck', 'proftpd', 'syslinux') },
137                          },                          },
138                  '10' => {                  '10' => {
139                              'centos'        => { map { $_ => 1 } ( 'samba' ) },
140                              'epel'          => { map { $_ => 1 } ( 'clamav') },
141                          },                          },
142                };                };
143    
144    # add here include only conditions
145    # for each version if a repo is listed here, only listed rpm will be considered
146    my $onlypkg = { '8' => {
147                              'remi' => { },
148                              'remisafe' => { },
149                            },
150                    '9'  => {
151                              'remi' => { },
152                            },
153                    '10' => {
154                              'remi' => { map { $_ => 1 } ( 'php-pear' )},
155                            },
156    };
157    
158  my ($stage) = sort { $repos->{$a}->{stage} <=> $repos->{$b}->{stage} } grep { $repos->{$_}->{stage} } keys %$repos;  my ($stage) = sort { $repos->{$a}->{stage} <=> $repos->{$b}->{stage} } grep { $repos->{$_}->{stage} } keys %$repos;
159  my ($devel1, $devel2) = sort { $repos->{$a}->{devel} <=> $repos->{$b}->{devel} } grep { $repos->{$_}->{devel} } keys %$repos;  my ($devel1, $devel2) = sort { $repos->{$a}->{devel} <=> $repos->{$b}->{devel} } grep { $repos->{$_}->{devel} } keys %$repos;
160  $devel2 ||= $devel1;  $devel2 ||= $devel1;
# Line 129  foreach my $ver ( sort { $a <=> $b } key Line 168  foreach my $ver ( sort { $a <=> $b } key
168      %sources = ();      %sources = ();
169      %base = ();      %base = ();
170    
171      @{$distrepo->{$ver}->{all_archs}} = ('noarch', @{$distrepo->{$ver}->{archs}});      @{$distrepo->{$ver}->{all_archs}} = ('noarch', @{$distrepo->{$ver}->{'archs'}});
172      if ( grep $_ eq 'i386', @{$distrepo->{$ver}->{archs}} ) {      if ( grep $_ eq 'i386', @{$distrepo->{$ver}->{'archs'}} ) {
173          push @{$distrepo->{$ver}->{all_archs}}, ('i586', 'i686');          push @{$distrepo->{$ver}->{all_archs}}, ('i586', 'i686');
174      }      }
175    
# Line 139  foreach my $ver ( sort { $a <=> $b } key Line 178  foreach my $ver ( sort { $a <=> $b } key
178          $bdir =~ s/~S/$distrepo->{$ver}->{repo}/;          $bdir =~ s/~S/$distrepo->{$ver}->{repo}/;
179          $bdir =~ s/~s/$ver/;          $bdir =~ s/~s/$ver/;
180          $bdir =~ s/~C/$distrepo->{$ver}->{centos}/;          $bdir =~ s/~C/$distrepo->{$ver}->{centos}/;
181          for my $arch ( @{$distrepo->{$ver}->{archs}}, 'SRPMS/' ) {          for my $arch ( @{$distrepo->{$ver}->{'archs'}}, 'SRPMS/' ) {
182              my $adir = $bdir;              my $adir = $bdir;
183              if ( $arch eq 'SRPMS/' ) {              if ( $arch eq 'SRPMS/' ) {
184                  $adir =~ s/~A.*//;                  $adir =~ s/~A.*//;
# Line 162  foreach my $ver ( sort { $a <=> $b } key Line 201  foreach my $ver ( sort { $a <=> $b } key
201              }              }
202              next unless -d $adir;              next unless -d $adir;
203              opendir DIR, $adir or next;              opendir DIR, $adir or next;
204              process_rpm("$adir$_", $ver, $baserepo->{$dir}) foreach readdir DIR;              my @txt = grep {  /\.rpm$/ && -f "$adir/$_" }  readdir DIR;
205              closedir DIR;              closedir DIR;
206                if (scalar(@txt)) {
207                   opendir DIR, $adir or next;
208                   process_rpm("$adir$_", $ver, $baserepo->{$dir})  foreach readdir DIR;
209                   closedir DIR;
210                } elsif ($baserepo->{$dir} eq 'epel') {
211                   # EPEL 7  uses a structure with subfolders of first letter of package
212                   find( { wanted => sub { process_rpm($_, $ver, $baserepo->{$dir}) if ($_ =~ m/\.rpm$/);}, no_chdir => 1, follow_fast => 1 }, $adir );
213                }
214              last unless $dir =~ m{~A};              last unless $dir =~ m{~A};
215          }          }
216      }      }
217      find( { wanted => sub { process_rpm($_, $ver, 'builds'); }, no_chdir => 1, follow_fast => 1 }, "$_" ) foreach @{$distrepo->{$ver}->{builds}};      find( { wanted => sub { process_rpm($_, $ver, 'builds'); }, no_chdir => 1, follow_fast => 1 }, "$_" ) foreach @{$distrepo->{$ver}->{'builds'}};
218      find( { wanted => sub { process_rpm($_, $ver, 'community'); }, no_chdir => 1, follow_fast => 1 }, $distrepo->{$ver}->{community} ) if $distrepo->{$ver}->{community};      find( { wanted => sub { process_rpm($_, $ver, 'community'); }, no_chdir => 1, follow_fast => 1 }, $distrepo->{$ver}->{'community'} ) if $distrepo->{$ver}->{'community'};
219    
220      foreach my $base ( sort keys %$rpms ) {      foreach my $base ( sort keys %$rpms ) {
221          next unless $rpms->{$base}->{rpms};          next unless $rpms->{$base}->{rpms};
# Line 394  foreach my $ver ( sort { $a <=> $b } key Line 441  foreach my $ver ( sort { $a <=> $b } key
441                  foreach my $a ( keys %{$acnt->{$r}->{$p}} ) {                  foreach my $a ( keys %{$acnt->{$r}->{$p}} ) {
442                      if ( $a eq 'noarch' ) {                      if ( $a eq 'noarch' ) {
443                          my $p2 = ${$acnt->{$r}->{$p}->{$a}}[0];                          my $p2 = ${$acnt->{$r}->{$p}->{$a}}[0];
444                          if ( ! $p2->{repos} && scalar @{$acnt->{$r}->{$p}->{$a}} != scalar @{$distrepo->{$ver}->{archs}} ) {                          if ( ! $p2->{repos} && scalar @{$acnt->{$r}->{$p}->{$a}} != scalar @{$distrepo->{$ver}->{'archs'}} ) {
445                              $p2->{oldrepo} = $p2->{repo};                              $p2->{oldrepo} = $p2->{repo};
446                              push @{$p2->{repos}}, $p2->{repo};                              push @{$p2->{repos}}, $p2->{repo};
447                          }                          }
# Line 452  foreach my $ver ( sort { $a <=> $b } key Line 499  foreach my $ver ( sort { $a <=> $b } key
499                      if ( $pkg->{src} ) {                      if ( $pkg->{src} ) {
500                          push @d, "${head}SRPMS/" . basename($pkg->{rpm}->filename);                          push @d, "${head}SRPMS/" . basename($pkg->{rpm}->filename);
501                      } elsif ( $pkg->{rpm}->arch eq 'noarch' ) {                      } elsif ( $pkg->{rpm}->arch eq 'noarch' ) {
502                          push @d, map { "${head}$_/$tail" . basename($pkg->{rpm}->filename) } @{$distrepo->{$ver}->{archs}};                          push @d, map { "${head}$_/$tail" . basename($pkg->{rpm}->filename) } @{$distrepo->{$ver}->{'archs'}};
503                        } elsif ($ver>=10 && $pkg->{rpm}->arch =~ m{^(i[356]86)$}  && $pkg->{arch} =~ m{^(x86_64)$}  ) {
504                           #add copy of i686 packages in x86_64 folder for SME>=10
505                           push @d, map { "${head}$_/$tail" . basename($pkg->{rpm}->filename) } @{$distrepo->{$ver}->{'archs'}};
506                      } elsif ( $pkg->{rpm}->arch =~ m{^(i[356]86)$} ) {                      } elsif ( $pkg->{rpm}->arch =~ m{^(i[356]86)$} ) {
507                          push @d, "${head}i386/$tail" . basename($pkg->{rpm}->filename);                          push @d, "${head}i386/$tail" . basename($pkg->{rpm}->filename);
508                      } else {                      } else {
# Line 496  foreach my $ver ( sort { $a <=> $b } key Line 546  foreach my $ver ( sort { $a <=> $b } key
546      }      }
547      foreach my $repo ( sort { $repos->{$b}->{prio} <=> $repos->{$a}->{prio} } keys %repochg ) {      foreach my $repo ( sort { $repos->{$b}->{prio} <=> $repos->{$a}->{prio} } keys %repochg ) {
548          next if $repos->{$repo}->{orig} || $repos->{$repo}->{stage};          next if $repos->{$repo}->{orig} || $repos->{$repo}->{stage};
549          foreach my $arch ( @{$distrepo->{$ver}->{archs}} ) {          foreach my $arch ( @{$distrepo->{$ver}->{'archs'}} ) {
550              my $dir = $distrepo->{$ver}->{repo} . "$repo/$arch";              my $dir = $distrepo->{$ver}->{repo} . "$repo/$arch";
551              $dir = qx(readlink -f $dir);              $dir = qx(readlink -f $dir);
552              if ( $dir ) {              if ( $dir ) {
# Line 527  foreach my $ver ( sort { $a <=> $b } key Line 577  foreach my $ver ( sort { $a <=> $b } key
577          }          }
578      }      }
579      unless ( $opts{t} ) {      unless ( $opts{t} ) {
580          foreach (@{$distrepo->{$ver}->{builds}})          foreach (@{$distrepo->{$ver}->{'builds'}})
581          {          {
582              finddepth(sub{rmdir}, "$_") if -d "$_";              finddepth(sub{rmdir}, "$_") if -d "$_";
583          }          }
584          finddepth(sub{rmdir}, $distrepo->{$ver}->{community}) if -d "$distrepo->{$ver}->{community}";          finddepth(sub{rmdir}, $distrepo->{$ver}->{'community'}) if -d "$distrepo->{$ver}->{'community'}";
585      }      }
586  }  }
587    
# Line 593  sub process_rpm { Line 643  sub process_rpm {
643      }      }
644      my ($src, $version, $release) = ($pkg->is_source_package ? $pkg->filename : $pkg->sourcerpm) =~ m{(?:^|/)([^/]*)-([^-]+)-([^-]+)\.\w+\.rpm$};      my ($src, $version, $release) = ($pkg->is_source_package ? $pkg->filename : $pkg->sourcerpm) =~ m{(?:^|/)([^/]*)-([^-]+)-([^-]+)\.\w+\.rpm$};
645      return if $skippkg->{$ver}->{$repo} && ( $skippkg->{$ver}->{$repo}->{$src} || $skippkg->{$ver}->{$repo}->{$pkg->name} );      return if $skippkg->{$ver}->{$repo} && ( $skippkg->{$ver}->{$repo}->{$src} || $skippkg->{$ver}->{$repo}->{$pkg->name} );
646        return if $onlypkg->{$ver}->{$repo} && !( $onlypkg->{$ver}->{$repo}->{$src} || $onlypkg->{$ver}->{$repo}->{$pkg->name} );
647      return if $repos->{$repo}->{base} && $nosig && $baserepo->{$ver}->{sign};      return if $repos->{$repo}->{base} && $nosig && $baserepo->{$ver}->{sign};
648      if ( $nosig && ! -f "$HOME/.rpmpass" ) { print " * missing signature (" .$pkg->filename . ")\n"; return; }      if ( $nosig && ! -f "$HOME/.rpmpass" ) { print " * missing signature (" .$pkg->filename . ")\n"; return; }
649      my $arch = 'unknown';      my $arch = 'unknown';

Legend:
Removed lines/characters  
Changed lines/characters
  Added lines/characters
admin@koozali.org
 ViewVC Help
Powered by ViewVC 1.2.1 RSS 2.0 feed