| Current Path : /var/www/web-klick.de/dsh/10_customer2017/1204__intel/ |
Linux ift1.ift-informatik.de 5.4.0-216-generic #236-Ubuntu SMP Fri Apr 11 19:53:21 UTC 2025 x86_64 |
| Current File : /var/www/web-klick.de/dsh/10_customer2017/1204__intel/scintegrate |
#!/opt/perl_5.8.8/bin/perl -w
# TODO
# - "nolock" option in ACS
use strict;
use Getopt::Long qw(GetOptions);
use Pod::Usage qw(pod2usage);
use ClearCase::CtCmd qw(cleartool);
use ClearCase::MtCmd qw(multitool);
use File::Temp qw(tempfile);
use File::Basename qw(basename);
use String::ShellQuote qw(shell_quote);
use MIME::Lite;
use Net::LDAP;
use Archive::Zip qw(:ERROR_CODES :CONSTANTS);
use POSIX qw(uname);
use lib "$ENV{WORKAREA}/stools/lib/perl";
use CCAPI;
use CmDbClient;
use CSgen;
# globals
our $VERSION = '1.8.0';
our $IntegUser = getlogin() || getpwuid($<);
our $Ok = 0;
# remember parent process PID
my $PID = $$;
# default options
my %opt = (
attach => '',
batch => 200,
freeze => CSgen->time_to_cstime(time),
'lock' => 1,
server => ($ENV{ACS_SERVER} || 'https://acs.lz.imc.local'),
smtp => 'smtp.imu.imc.local',
timeout => 5, # minutes
verbose => 0
);
unless(GetOptions(\%opt, qw(
acs=s
attach=s
baseline=s
batch=i
check!
freeze=s
full!
help!
ibranch=s
integrator=s
lock!
man!
nobaseline
onebaseline!
oneregression!
regression=s
server=s
setup!
show!
smtp=s
teelog!
timeout=i
touch!
verbose|v+
vobs=s
ziplog!
))) {
$PID = 0; # no END processing
pod2usage(-exitval => 1, -verbose => 0);
}
if($opt{help}) {
print "\n$0 Version $VERSION\n\n";
$PID = 0; # no END processing
pod2usage(-exitval => 0, -verbose => 0);
}
if($opt{man}) {
$PID = 0; # no END processing
pod2usage(-exitval => 0, -verbose => 3);
}
# immediately flush STDOUT
$| = 1;
# for time measurements
my ($_ts, $_time_task);
# need this for creating the label types
our @AVOBs = split(/[:;]+/, $ENV{CLEARCASE_AVOBS}||'');
# determine VOBs to process
our $CCAPI = CCAPI->new(
VERBOSITY => $opt{verbose},
$opt{vobs} ? (VOBS => [split(/[\s,:;]+/, $opt{vobs})]) : (),
LOG_INFO => sub { log_both(join('', @_)) },
LOG_WARN => sub { log_both(join('', @_)) } );
unless($CCAPI->check_vobs) {
die "[ERROR] No VOBs identified... please use the -vobs option.\n";
}
# Flexible integ branch concept.
# TODO insert a (fallback) .../<integbranch>/LATEST rule to branch off from?
our $IntegrationBranch; # TODO get this from ACS's cnd: definition (first part)
# buffers for messages
our @DeveloperMessage;
our @IntegratorMessage;
our $HaveCheckouts = 0;
my $exit_status = 0;
# only show branches ready-for-integration
if($opt{show} || $opt{check}) {
$IntegrationBranch = _get_integration_branch($opt{ibranch});
my @branches = _get_integratable_branches($IntegrationBranch);
print "INFO: These branches are marked as ready-for-integration to integration branch '$IntegrationBranch':\n";
if(@branches) {
print " ",join(', ',sort @branches),"\n\n";
} else {
print " <none>\n";
}
@IntegratorMessage = (); # avoid email
$Ok = 1;
exit($opt{check} ? scalar(@branches) : 0);
}
# preparation only
if($opt{setup}) {
exit 1 unless $CCAPI->check_vob_owner(USER => $IntegUser);
$CCAPI->setup_vobs();
@IntegratorMessage = (); # no mail
$Ok = 1;
exit 0;
}
# ClearCase View sanity check
my $CurrentView = $CCAPI->get_current_view;
unless($CurrentView) {
log_integ("ERROR: Cannot determine current view, cannot continue.\n");
$exit_status = 2;
goto EndIntegration;
}
# startup message
my @uname = uname();
if($uname[0] =~ /linux/i) {
if(open(IN, "</etc/redhat-release")) {
my $line = <IN>;
chomp $line;
$uname[2] = $line;
close(IN);
}
}
my $StartTime = localtime();
my @VOBs = $CCAPI->get_vobs;
log_both(<<"EOT");
INFO: This is scintegrate v$VERSION
Host: $uname[1] $uname[4] $uname[0]/$uname[2]
Time: $StartTime
View: $CurrentView
VOBs: @VOBs
EOT
# sanity check - no checkouts to current view
my @cos = $CCAPI->get_view_checkouts();
if(@cos) {
log_integ("ERROR: There are checkouts to the current view:\n ".join("\n ",@cos)."\n");
$exit_status = 2;
goto EndIntegration;
}
# TODO cleanup private files here?
# check that current user is VOB owner
unless($CCAPI->check_vob_owner(USER => $IntegUser)) {
$exit_status = 2;
goto EndIntegration;
}
# protect against CTRL-C - we want to clean up before exiting
$SIG{INT} = sub {
die "\nWARNING: Caught CTRL-C, exiting.\n";
};
# which branches to merge?
$Ok = 1; # we assume OK
our @BranchQueue;
unless(@ARGV) {
# lock the integration branches (all VOBs)
$IntegrationBranch = _get_integration_branch($opt{ibranch});
@BranchQueue = _get_integratable_branches($IntegrationBranch);
} else {
@BranchQueue = @ARGV;
$IntegrationBranch = _get_integration_branch($opt{ibranch},1);
# check default deliver target and warn about inconsistencies
foreach my $br (@BranchQueue) {
my $ibranch = $CCAPI->get_integration_branch(BRANCH => $br);
if($ibranch && !$IntegrationBranch) {
$IntegrationBranch = $ibranch; # the first wins
log_integ("INFO: Integrating branch '$br' to '$ibranch'.\n");
}
elsif($IntegrationBranch && $ibranch && $ibranch ne $IntegrationBranch) {
log_integ(<<"EOT");
WARNING: The default integration branch for task branch '$br' is '$ibranch'.
We're however integrating to '$IntegrationBranch' now.
EOT
}
} # loop branches
unless($IntegrationBranch) {
$IntegrationBranch = 'main'; # fallback if not defined
log_integ("INFO: Falling back to integration branch 'main'.\n");
}
}
# warn about any checkouts on integ branch
$CCAPI->check_branch_checkouts(BRANCH => $IntegrationBranch);
# do we have anything to do?
unless(@BranchQueue) {
log_integ("INFO: No integratable branches specified/found.\n");
$exit_status = 0;
goto EndIntegration;
}
# lock the integration branch
my @integrators = _get_integrators($IntegrationBranch);
_start_time("lock integration branch");
my $lockstat = $CCAPI->lock_integration_branch(BRANCH => $IntegrationBranch,
COMMENT => "Integration in progress by user $IntegUser (@integrators)",
USER => $IntegUser);
_end_time();
unless($lockstat) {
$exit_status = 2;
goto EndIntegration;
}
# get the server connection
my ($acs_host, $acs_port) = $opt{server} =~ /^(https?:[^:]+):?(\d*)$/;
my $CmDbClient = CmDbClient->new(
VERBOSITY => $opt{verbose},
HOST => $acs_host, PORT => $acs_port,
LOG_INFO => sub { log_both(join('', @_)) },
LOG_WARN => sub { log_both(join('', @_)) } );
# check for IntegBranch/LATEST not marked with recommended Baseline label
our @IntegLatest;
our $IntegHead = $CCAPI->get_recommended_baseline(BRANCH => $IntegrationBranch);
my @head_labels = $CCAPI->get_baseline_labels(BASELINE => $IntegHead);
my $brv = $IntegrationBranch eq 'main' ? '/main' : ".../$IntegrationBranch";
_start_time("HEAD vs. LATEST");
log_integ("INFO: Checking for differences between $IntegrationBranch/LATEST and head baseline '$IntegHead' ...\n");
my ($stat,$out,$err) = cleartool(qw(find -avob -vis), -element => "brtype($IntegrationBranch)",
-version => join(' && ', "version($brv/LATEST)", map { "!lbtype($_)" } @head_labels),
'-print');
_end_time();
if($stat) {
log_integ("ERROR: Cannot run find:\n$out$err");
} else {
chomp $out;
@IntegLatest = split(/[\r\n]+/, $out);
if(@IntegLatest) {
log_both("WARNING: The following versions are on $IntegrationBranch/LATEST, but do not carry the HEAD label '$IntegHead':\n ".
join("\n ", @IntegLatest)."\n\n");
} else {
log_both("INFO: There are no versions on $IntegrationBranch/LATEST that do not carry the HEAD label '$IntegHead'. Good!\n");
}
}
# integration ACS specified on command line
my ($c_user,$c_project,$c_acs,$c_rev);
if($opt{acs}) {
($c_user,$c_project,$c_acs,$c_rev) = $CmDbClient->parse_acs_definition($opt{acs});
unless($c_acs) {
die "[ERROR] Please check the -acs option.\n";
} else {
log_both("INFO: Using integration ACS '$opt{acs}' specified on command line.\n");
}
}
# loop over task branches to be integrated
my @success_integrations;
our @AllLogfiles;
our %OpenCheckouts;
our $LastACS;
foreach my $branch (@BranchQueue) {
# get the config spec from ACS
my $branch_acs = $CCAPI->get_branch_acs(BRANCH => $branch);
my ($user,$project,$acs,$rev) = ($c_user,$c_project,$c_acs,$c_rev);
if($branch_acs) {
my ($u,$p,$a,$r) = $CmDbClient->parse_acs_definition($branch_acs);
$user = $u unless defined $user;
$project = $p unless defined $project;
$acs = $a unless defined $acs;
$rev = $r unless defined $rev;
}
# now we have all
my $new_acs = $CmDbClient->write_acs_definition(USER => $user, PROJECT => $project,
ACS => $acs, REVISION => $rev);
# get the ACS details
my $ACS = $CmDbClient->get_ACS(PROJECT => $project, ACS => $acs, REV => $rev,
($user ? (USER => $user) : ()) );
unless($ACS) {
log_both("[ERROR] Cannot retrieve ACS '$new_acs' from server.\n");
$exit_status = 1;
goto End_Branch;
}
$LastACS = $ACS;
log_both("INFO: Using ACS '$new_acs' for integration of branch '$branch'.\n");
# component and domain definitions
my ($ACS_cnd,$CnD_name, $CnD_rev);
$ACS_cnd = $ACS->getCnDSelector();
if($ACS_cnd && $ACS_cnd =~ /^([^.]+)\.([^.]+)$/) {
($CnD_name, $CnD_rev) = ($1,$2);
} else {
log_both("[ERROR] ACS does not contain a valid Component/Domain definition.\n");
$exit_status = 1;
goto End_Branch;
}
my $CnD = $CmDbClient->get_CnD(PROJECT => $project, REV => [ $CnD_name => $CnD_rev ]);
unless($CnD) {
log_both("[ERROR] Cannot retrieve Component/Domain definition '$CnD_name.$CnD_rev' from server.\n");
$exit_status = 1;
goto End_Branch;
}
# we may need a config spec include file
my ($inc_file, $inc_selector) = $ACS->getIncludeFileSelector;
my $CSgen = CSgen->new(ACS => $ACS, CND => $CnD, INTEGRATE => $IntegrationBranch,
INCLUDE_FILE => $inc_file, INCLUDE_SELECTOR => $inc_selector,
TIMESTAMP => $opt{freeze},
VERBOSITY => $opt{verbose},
LOG_INFO => sub { log_both(join('', @_)) },
LOG_WARN => sub { log_both(join('', @_)) } );
my @config_spec = $CSgen->generate_CS();
# set the config spec
unless($CCAPI->set_config_spec(CONTENT => join("\n", @config_spec)."\n")) {
$exit_status = 2;
goto EndIntegration;
}
### get the mastership of the task branch
# this fails when there are remote checkouts
# TODO no check yet for local checkouts on the task branch
# - do we want this anyway?
# TODO should the task branch be locked, -nuser current user?
# TODO time stamp on task branch, to allow continued development?
# warn about checkouts on task branch
$CCAPI->check_branch_checkouts(BRANCH => $branch);
_start_time("get taskbranch mastership");
my $stat = $CCAPI->get_branch_mastership(BRANCH => $branch);
_end_time();
unless($stat) {
log_both(<<"EOT");
An error occurred while transferring the mastership of branch '$branch'.
This prevents the integration from proceeding.
Please check the errors above, fix the root cause and deliver again.
EOT
# TODO reject branch (where possible) to prevent same issue
# (note: requires that 'rejected' status overrides 'ready')
$exit_status = 1;
goto End_Branch;
}
### start the merge
# - first all directories, try checkout
# => RFM if failures, redo
# - next the files
# => abort if conflict, tell user to rebase
my %current_checkouts;
log_both("INFO: Starting integration of task branch '$branch' to integration branch '$IntegrationBranch' ...\n");
my $ok = _merge_directories($branch, \%current_checkouts) &&
_merge_files($branch, \%current_checkouts);
my $reject_status;
unless($ok) {
# merge failed, cancel checkouts
# TODO try to selectively cancel the checkouts
$reject_status = _reject_branch($branch, \%current_checkouts);
unless($opt{oneregression}) {
log_both("ERROR: Integration of task branch '$branch' aborted because of merge conflicts.\n");
goto End_Branch;
}
}
# special case: merge everything, then run regression
if($opt{oneregression}) {
if($ok) {
# the merge succeeded - remember our checkouts so far
foreach(keys %current_checkouts) {
$OpenCheckouts{$_}++;
}
push(@success_integrations, $branch);
next; # proceed with next branch
}
elsif($reject_status) {
# it was possible to uncheckout the current merge only,
# so we can proceed with the next branch.
# send the email to the developer (TODO text per branch):
goto End_Branch;
}
# we have overlapping change sets and merge conflicts - abort
$exit_status = 1;
@success_integrations = ();
last; # do not proceed in the loop here
# TODO alternative: cancel all(!) checkouts, and continue the loop
}
### run regression
my @reg_logfiles;
# smoke test script from command line or ACS
my $regression = _get_regression($opt{regression}, $ACS);
if($ok && $regression) {
my $reg_logfile;
($ok, $reg_logfile) = _run_regression($regression, $branch);
push(@reg_logfiles, $reg_logfile);
unless($ok) {
# regression failed, cancel checkouts
_reject_branch($branch);
$exit_status = 1;
goto End_Branch;
}
}
my $msg = $regression ? ' and successfully tested.' : '.';
log_both(<<"EOT");
INFO: Merge of branch '$branch' to '$IntegrationBranch' completed$msg
EOT
# merge the VOB list from CCAPI and ACS for labeling
$CCAPI->add_vobs($CSgen->get_VOBs);
goto Checkin_Branch if $opt{nobaseline};
if($opt{onebaseline}) {
# apply the label now in case of incremental (based on checkouts)
my $baseline;
unless($opt{full}) {
($ok,$baseline) = _create_baseline([$branch], 1);
# reuse the baseline name
$opt{baseline} = $baseline if $ok;
if($ok) {
log_both("INFO: Recorded the integration of branch '$branch' in baseline '$baseline' on integration branch '$IntegrationBranch'.\n");
} else {
# baseline creation not ok
$baseline ||= '<none>';
log_both(<<"EOT");
ERROR: Failed to record the integration in baseline '$baseline'. Cancelling the merge.
EOT
_reject_branch($branch);
$exit_status = 1;
goto End_Branch;
}
} # incremental baseline
# proceed with checkin
} else { # !onebaseline
### create individual baseline
my $baseline;
# create baseline for one branch integration - full or incremental
($ok,$baseline) = _create_baseline([$branch], !$opt{full});
### close task branch
if($ok) {
log_both("INFO: Recorded the integration of branch '$branch' in baseline '$baseline' on integration branch '$IntegrationBranch'.\n");
# metadata: record baseline with branch, lock branch
# ignore the return value - errors in closing do not make the integ fail
_close_branch($branch);
_close_baseline($baseline,$branch) if($baseline);
} else {
$baseline ||= '<none>';
log_both(<<"EOT");
ERROR: Merged branch '$branch' to integration branch '$IntegrationBranch',
but problems occured while creating baseline '$baseline', merge will be rejected.
Please check the issues above, and contact the project integrator.
EOT
log_integ("WARNING: Will not close branch and baseline.");
_reject_branch($branch);
$exit_status = 1;
goto End_Branch;
}
} # else onebaseline
Checkin_Branch:
### checkin the merges
# TODO lock integration branch on checkin failures?
$ok = _checkin_all($branch);
if($ok) {
log_both("INFO: Successfully checked in merged elements for branch '$branch'.\n");
push(@success_integrations, $branch);
} else {
log_both("ERROR: Checkin of merge results failed - please contact the integrator.");
}
End_Branch:
### compose message to developer
_send_developer_mail($IntegrationBranch, $ok, \@reg_logfiles, $branch);
push(@AllLogfiles, @reg_logfiles);
$Ok = 0 unless($ok); # there was a failure
} # loop over integratable branches
if(@success_integrations && $opt{oneregression}) {
# there were successful merges
# run regression over all merges
my $ok = 1;
my $regression = _get_regression($opt{regression}, $LastACS);
if($regression) {
my $reg_logfile;
($ok, $reg_logfile) = _run_regression($opt{regression}, @success_integrations);
push(@AllLogfiles, $reg_logfile);
unless($ok) {
# regression failed, cancel checkouts
$exit_status = 1;
if(@success_integrations == 1) {
# we know precisely why it failed :-)
_reject_branch($success_integrations[0]);
_send_developer_mail($IntegrationBranch, 0, [ $reg_logfile ], $success_integrations[0]);
} else {
_cancel_checkouts();
log_both(<<"EOT");
ERROR: Regression failed on merged branches:
@success_integrations
It is unclear which branch caused the failure, please check manually.
EOT
_send_developer_mail($IntegrationBranch, 0, [ $reg_logfile ], @success_integrations);
}
@success_integrations = ();
} # not ok
}
# need to create incremental baseline here
if($ok && !$opt{full} && !$opt{nobaseline}) {
my $baseline;
($ok,$baseline) = _create_baseline(\@success_integrations, 1);
# reuse the baseline name
$opt{baseline} = $baseline if $ok;
if($ok) {
foreach(@success_integrations) {
log_both("INFO: Recorded the integration of branch '$_' in baseline '$baseline' on integration branch '$IntegrationBranch'.\n");
}
} else {
# baseline creation not ok
$baseline ||= '<none>';
log_both(<<"EOT");
ERROR: Failed to record the integration in baseline '$baseline'. Cancelling all merges:
@success_integrations
EOT
$exit_status = 1;
if(@success_integrations == 1) {
# we know precisely why it failed :-)
_reject_branch($success_integrations[0]);
_send_developer_mail($IntegrationBranch, 0, [], $success_integrations[0]);
} else {
_cancel_checkouts();
log_both("WARNING: It is unclear which branch caused the failure, please check manually.");
_send_developer_mail($IntegrationBranch, 0, [], @success_integrations);
}
@success_integrations = ();
}
}
if($ok) {
### checkin all merges
$ok = _checkin_all(@success_integrations);
# TODO send message to developers if this fails?
}
# proceed with baseline - if OK
$opt{onebaseline} = $ok;
} # oneregression
if(@success_integrations && $opt{onebaseline}) {
# incremental baseline labels have been already created
my ($ok,$baseline) = (1, $opt{baseline});
if($opt{full} && !$opt{nobaseline}) {
# TODO this requires an ibranch/LATEST config spec!!!
($ok,$baseline) = _create_baseline(\@success_integrations, 0);
if($ok) {
foreach(@success_integrations) {
log_both("INFO: Recorded the integration of branch '$_' in baseline '$baseline' on integration branch '$IntegrationBranch'.\n");
}
} else { # error occurred
# baseline creation not ok
my $btxt = $baseline || '<none>';
log_both(<<"EOT");
ERROR: Failed to record the integration in baseline '$btxt'. Cancelling all merges:
@success_integrations
EOT
$exit_status = 1;
if(@success_integrations == 1) {
# we know precisely why it failed :-)
_reject_branch($success_integrations[0]);
_send_developer_mail($IntegrationBranch, 0, [], $success_integrations[0]);
} else {
_cancel_checkouts();
log_both("WARNING: This is a general failure, rejecting all branches. Please check manually.");
_send_developer_mail($IntegrationBranch, 0, [], @success_integrations);
}
@success_integrations = ();
}
}
### close task branch
if($ok) {
foreach my $branch (@success_integrations) {
# record baseline with branch, lock branch
# ignore the return values
_close_branch($branch);
if($baseline && !$opt{nobaseline}) {
log_both("INFO: Recorded the integration of branch '$branch' in baseline '$baseline' on integration branch '$IntegrationBranch'.\n");
}
}
_close_baseline($baseline, @success_integrations) if($baseline && !$opt{nobaseline});
log_both(<<"EOT");
INFO: Integration of the following task branches successfully completed:
@success_integrations
EOT
if(!$baseline || $opt{nobaseline}) {
log_both("INFO: Changes are integrated on the integration branch $IntegrationBranch, but no baseline was created.\n");
}
}
# send developer messages!
_send_developer_mail($IntegrationBranch, $ok, \@AllLogfiles, @success_integrations);
} # onebaseline
EndIntegration:
log_both("INFO: Integration finished, exit status = $exit_status.\n");
exit $exit_status;
##############################################################################
sub _get_integration_branch # REVIEWED
{
my ($opt,$nomain) = @_;
if($opt) {
if($ENV{INTEGRATION_BRANCH} && $ENV{INTEGRATION_BRANCH} ne $opt) {
log_integ("INFO: Will use integration branch '$opt', overriding environment INTEGRATION_BRANCH=$ENV{INTEGRATION_BRANCH}\n");
}
return $opt;
}
elsif($ENV{INTEGRATION_BRANCH}) {
log_integ("INFO: Will use integration branch from INTEGRATION_BRANCH=$ENV{INTEGRATION_BRANCH}\n");
return $ENV{INTEGRATION_BRANCH};
}
return if $nomain;
log_integ("INFO: Will use default integration branch 'main'\n");
return 'main';
}
sub _get_integratable_branches # REVIEWED
{
my ($ibranch) = @_;
my %branches;
log_integ("INFO: Looking for branches integratable to '$ibranch'...\n");
_start_time("find integratable task branches");
my @branches = $CCAPI->get_child_branches(BRANCH => $ibranch, STATE => 'ready');
my $brs = join(',', @branches) || '<none>';
_end_time();
log_integ("INFO: Found integratable branches: $brs\n");
return @branches;
}
sub _merge_directories # REVIEWED
{
my ($branch, $checkouts_ref) = @_;
# TODO this could be a way to restrict the delivery by time stamp, which
# would allow to continue work on the task branch, and integration would
# just merge the prior versions (when the TASK_STATE attribute with value
# "ready" was attached).
# This way one could avoid to create deliver labels at "ccbranch -deliver":
# ct findmerge -avob -fver '.../plogstie_LUP_COSIM/{\!created_since(01-Apr-2010)}' -print
# start loop - could be that there are dirs to be merged
my $status = 1;
my $iter = 0;
my @vob_arg = qw(-avob);
_start_time("merge directories");
my @voblist = $CCAPI->get_vobs;
while(++$iter) {
# first determine the directories to be merged
log_integ("INFO: Checking for required directory merges (iteration $iter)...\n");
my ($stat,$out,$err) = cleartool('findmerge', @vob_arg, -type => 'd', -fver => ".../$branch/LATEST", '-print', '-nxn');
if($stat) {
log_integ("ERROR: Cannot run findmerge for directories on branch '$branch'.\n$out$err");
$status = 0;
last;
}
chomp $out;
# we must sort to process the leafs first - the leaf directory might be renamed
my @dirs = sort { $b cmp $a } split(/[\n\r]+/, $out);
# check out, with eventual request-for-mastership
if(@dirs) {
log_both("INFO: Need to checkout ".scalar(@dirs)." directory/ies.\n");
my @todo = @dirs;
while(my @list = splice(@todo, 0, $opt{batch})) {
my $cmd = qq(ccco -timeout $opt{timeout} -comment "merge of task-branch '$branch'" ).
shell_quote(@list);
my $out = `$cmd 2>&1`;
if($?) {
$status = 0;
}
# TODO: detection of non-trivial merges
# cleartool: Warning: Version checked out ("/main/2") is different from version selected by view before checkout ("/main/1").
# Checked out "mynewfile.txt" from version "/main/2".
my ($old,$new);
foreach(split(/[\r\n]+/, $out)) {
if(/Warning: Version checked out \("([^"]+)"\) is different from version selected by view before checkout \("([^"]+)"\)\./) {
($old,$new) = ($2,$1);
}
elsif(/Checked out "([^"]+)" from version "([^"]+)"\./) {
$HaveCheckouts++;
if($old && $new) {
log_both("[WARNING] Directory '$1' has more recent version '$new' than the one selected by baseline '$old'. Will merge into '$2'.\n");
}
$old = $new = undef;
}
}
$HaveCheckouts++; # TODO remove after ccco fix
log_integ($out);
} # loop parts
# remember the checkouts
foreach my $dir (@dirs) {
$checkouts_ref->{$dir} = 0; # this was checked out
}
if($status) {
log_both("INFO: Successfully checked out directories.\n");
} else {
log_both("ERROR: Could not check out all directories for merging '$branch'.\n");
$status = 0;
last;
}
} else {
log_integ("INFO: No more directories to be merged.\n");
last; # no more dirs to merge
}
# now we are ready to merge
my @conflicts;
my @errors;
my %old_name;
foreach my $dir (@dirs) {
my ($stat,$out,$err) = cleartool('desc', -fmt => '%On\n', "$dir\@\@");
if($stat == 0) {
chomp $out;
$old_name{"oid:$out\@$dir"} = $dir;
} else {
log_integ("ERROR: Cannot get oid for $dir\@\@:\n$out$err\n");
}
($stat,$out,$err) = cleartool(qw(merge -abort),
'-to' => $dir, '-version' => ".../$branch/LATEST");
unless($stat) {
$checkouts_ref->{$dir} = 1; # this was merged OK
} else {
# cleartool: Error: Already a merge from version "/main/mybranch/1" to "automatic.txt".
if($err =~ /Error: Already a merge from version "[^"]*" to "[^"]*"/) {
$stat = 0; # this is OK
$checkouts_ref->{$dir} = 1; # merged OK
}
elsif($out =~ /^\*\*\* No Automatic Decision Possible/m ||
$err =~ /Warning: \*\*\* Aborting\.\.\./) {
# conflict
my $ancestor = '';
if($out =~ /^\s*<<< file 1: .*?\@\@(\S+)/m) {
$ancestor = " (ancestor: $1)";
}
push(@conflicts, " $dir$ancestor\n");
} # merge conflict
else {
# some other unspecified error
push(@errors, "> $dir\n$out$err\n");
}
} # if error
} # loop merge dirs
# take care of renamed dirs
foreach my $oid (keys %old_name) {
my ($stat,$out,$err) = cleartool('desc', -fmt => '%En\n', $oid);
if($stat) {
log_integ("ERROR: Cannot describe $oid:\n$out$err\n");
next;
}
chomp $out;
my $new = $out;
my $old = $old_name{$oid};
if($new ne $old) {
# the directory was renamed during the merge; need to update the
# checkouts list
my $val = delete $checkouts_ref->{$old};
$checkouts_ref->{$new} = $val if defined $val;
log_integ("INFO: Noticed directory rename: $old => $new\n");
}
} # end renamed dirs
if(@conflicts) {
log_both(<<"EOT");
There are merge conflicts when integrating task branch '$branch'
on the following directories:
@conflicts
You have to rebase your task branch to the most recent baseline and
resolve the conflict on the task branch. After that (and any required
testing) you can start the integration again.
EOT
my $recommended = $CCAPI->get_recommended_baseline(BRANCH => $IntegrationBranch);
if($recommended) {
log_both(<<"EOT");
The latest recommended baseline on the '$IntegrationBranch' branch is:
$recommended
EOT
}
$status = 0;
}
if(@errors) {
log_both(<<"EOT");
Errors occurred while merging directories from branch '$branch':
@errors
The integration was therefore rejected, and has to be restarted again
after fixing the root cause of these issues.
EOT
$status = 0;
last;
}
log_both("INFO: Successfully merged ".scalar(@dirs)." directory/ies.\n");
# ... recurse - we might have more dirs to merge
# but only into the VOBs where dirs were merged
my @newlist;
foreach my $vob (@voblist) {
if(grep(m{^\Q$vob\E(?:/|\\|$)}, @dirs)) {
push(@newlist, $vob);
}
}
@voblist = @newlist;
@vob_arg = (@newlist, '-all');
} # outer loop
_end_time();
if($status) {
log_both("INFO: Successfully checked out and merged all directories.\n");
return 1;
}
return 0;
}
sub _merge_files # REVIEWED
{
my ($branch, $checkouts_ref) = @_;
_start_time("merge files");
# first determine which elements need to be merged
my ($stat,$out,$err) = cleartool('findmerge', '-avob', -fver => ".../$branch/LATEST", '-print', '-nxn');
if($stat) {
_end_time();
log_integ("ERROR: Cannot run findmerge for all elements from branch '$branch'.\n$out$err");
return 0;
}
chomp $out;
my @elements = split(/[\n\r]+/, $out);
# check out, with eventual request-for-mastership
my $status = 1;
if(@elements) {
log_both("INFO: Need to checkout ".scalar(@elements)." element(s).\n");
my @todo = @elements;
while(my @list = splice(@todo, 0, $opt{batch})) {
my $cmd = qq(ccco -timeout $opt{timeout} -comment "merge of task-branch '$branch'" ).
shell_quote(@list);
my $out = `$cmd 2>&1`;
if($?) {
$status = 0;
}
my ($old,$new);
foreach(split(/[\r\n]+/, $out)) {
if(/Warning: Version checked out \("([^"]+)"\) is different from version selected by view before checkout \("([^"]+)"\)\./) {
($old,$new) = ($2,$1);
}
elsif(/Checked out "([^"]+)" from version "([^"]+)"\./) {
$HaveCheckouts++;
if($old && $new) {
log_both("[WARNING] Element '$1' has more recent version '$new' than the one selected by baseline '$old'. Will merge into '$2'.\n");
}
$old = $new = undef;
}
}
log_integ($out);
$HaveCheckouts++; # remove after ccco fix
} # loop parts
# TODO merge this into the loop above
foreach(@elements) {
$checkouts_ref->{$_} = 0; # this was checked out
}
if($status) {
log_both("INFO: Successfully checked out all elements for merging '$branch'.\n");
} else {
_end_time();
log_both("ERROR: Could not check out all elements for merging '$branch'.\n");
return 0;
}
} # have elements
# now we are ready to merge
my @conflicts;
my @errors;
foreach my $element (@elements) {
if($opt{touch}) {
my $cmd = shell_quote('touch', $element);
my $out = `$cmd 2>&1`;
if($?) {
log_integ("ERROR: Failed to touch merge destination '$element':\n$out");
}
}
# TODO restrict LATEST with timestamp?
my ($stat,$out,$err) = cleartool(qw(merge -abort),
'-to' => $element, '-version' => ".../$branch/LATEST");
unless($stat) {
$checkouts_ref->{$element} = 1; # this was merged OK
} else {
# cleartool: Error: Already a merge from version "/main/mybranch/1" to "automatic.txt".
if($err =~ /Error: Already a merge from version "[^"]*" to "[^"]*"/) {
$stat = 0; # this is OK
$checkouts_ref->{$element} = 1; # this was merged OK
}
elsif($out =~ /^\*\*\* No Automatic Decision Possible/m ||
$err =~ /Warning: \*\*\* Aborting\.\.\./) {
# conflict
# <<< file 1: /vobs/smartilu-LUP-fe_dig/units/tp_com/source/vhdl/rtl/tp-e.vhd@@/main/2
my $ancestor = '';
if($out =~ /^\s*<<< file 1: .*?\@\@(\S+)/m) {
$ancestor = " (ancestor: $1)";
}
push(@conflicts, " $element$ancestor\n");
} # merge conflict
else {
# some other unspecified error
push(@errors, "> $element\n$out$err");
}
} # if error
} # loop elements
if(@conflicts) {
log_both(<<"EOT");
There is at least one merge conflict when integrating task branch '$branch'.
You have to rebase your task branch to the most recent baseline and
resolve the conflict on the task branch. After that (and any required
testing) you can start the integration again.
List of elements with merge conflicts:
@conflicts
EOT
my $recommended = $CCAPI->get_recommended_baseline(BRANCH => $IntegrationBranch);
if($recommended) {
log_both(<<"EOT");
The latest recommended baseline on the integration branch '$IntegrationBranch' is:
$recommended
EOT
}
$status = 0;
}
_end_time();
if(@errors) {
log_both(<<"EOT");
Errors have occurred while merging files from branch '$branch':
@errors
The integration was therefore rejected, and has to be restarted again
after fixing the root cause of these issues.
EOT
$status = 0;
} else {
log_both("INFO: Successfully merged all elements.\n");
}
return $status;
}
sub _reject_branch # REVIEWED
{
my ($branch, $checkouts_ref) = @_;
log_both("INFO: Branch '$branch' is rejected because of previous errors.\n");
foreach my $vob ($CCAPI->get_vobs) {
my ($stat,$out,$err) = cleartool(qw(mkattr -replace TASK_STATE "rejected"), "brtype:$branch\@$vob");
if($stat) {
log_both("ERROR: Cannot reject branch '$branch' in VOB '$vob':\n$out$err\n");
return 0;
}
}
# cancel checkouts now
_cancel_checkouts($checkouts_ref);
}
sub _get_regression
{
my ($opt,$acs) = @_;
unless(defined $opt) {
$opt = $acs->getIntegrationCommand;
if($opt) {
$opt = CSgen->get_VOB_tag(PATH => $opt);
$opt =~ s#\${(\w+)}#$ENV{$1}#eg;
$opt =~ s#\$(\w+)#$ENV{$1}#eg;
}
}
return $opt;
}
sub _run_regression # REVIEWED
{
my ($script, @branches) = @_;
_start_time("regression run");
my $reg_status = 1;
my ($logfh,$log) = tempfile("regression_XXXXXXXX", TMPDIR => 1, SUFFIX => '.log');
log_integ("INFO: Starting regression script, logfile $log on host $uname[1]:\n $script\n");
my $reg_exit;
if($opt{teelog}) {
if(open(SCRIPT, "($script) 2>&1 |")) {
while(<SCRIPT>) {
print $_;
print $logfh $_;
}
close(SCRIPT);
$reg_exit = $?;
} else {
log_both("ERROR: Cannot launch script '$script': $!\n");
$reg_exit = 255 << 8;
}
} else {
system("($script) >$log 2>&1");
$reg_exit = $?;
}
close($logfh);
_end_time();
if($reg_exit) {
# failure
my $exit = $reg_exit >> 8;
my $sig = $reg_exit & 0x7f;
log_both(<<"EOT");
ERROR: The regression command
$script
returned an error status (exit=$exit, signal=$sig).
The regression failed, and thus the integration of task branch(es)
@branches
will be canceled. Check the attached log file, fix the problem and
deliver again.
EOT
$reg_status = 0;
} else {
# success
log_both(<<"EOT");
INFO: The regression command
$script
successfully completed. See also attached log.
INFO: Proceeding with integration of task branch(es):
@branches
EOT
}
# compress the logfile
if($opt{ziplog}) {
my $zipname = "$log.zip";
my $zip = Archive::Zip->new();
my $file_member = $zip->addFile($log, basename($log));
my $stat = $zip->writeToFileNamed($zipname);
if($stat == AZ_OK) {
unlink($log);
$log = $zipname;
} else {
unlink($zip);
log_integ("ERROR: Cannot compress regression log file '$log' to zip file '$zipname', error code $stat. Will fall back to uncompressed log file.\n");
}
}
return($reg_status,$log);
}
sub _checkin_all # REVIEWED
{
my (@branches) = @_;
_start_time("checkin");
log_both("INFO: Checking in merge results...\n");
my ($stat,$out,$err) = cleartool(qw(lsco -cview -avob -short));
if($stat) {
_end_time();
log_integ("ERROR: Cannot list checkouts in all VOBs:\n$out$err");
return 0;
}
chomp $out;
my @cos = reverse sort split(/[\r\n]+/, $out);
my $status = 1;
while(my @list = splice(@cos, 0, $opt{batch})) {
# TODO in case of identical, we could also check for labels and
# hyperlinks on the checkout, and transfer them to the predecessor
# version; this is the more simple solution (-identical)
($stat,$out,$err) = cleartool(qw(ci -ptime -identical), '-c' => "Merged branch(es) @branches", @list);
if($stat) {
log_integ("ERROR: Problems while checking in:\n$out$err");
$status = 0;
}
} # loop cos
_end_time();
log_both("INFO: Done checking in at ".localtime().".\n");
return $status;
}
sub _create_baseline # REVIEWED
{
my ($branches_ref, $incremental) = @_;
_start_time("make baseline");
my $label;
my $text;
unless($incremental) {
$text = "fully";
my %opts = (BRANCH => $IntegrationBranch,
COMMENT => "integrated branch(es) @$branches_ref",
ADDITIONAL_VOBS => \@AVOBs);
if($opt{baseline}) {
$opts{LABEL} = $opt{baseline};
}
if(!$opt{full} && $opt{onebaseline}) {
# if the same element is modified on several task branches, we may need to replace
$opts{REPLACE} = 1;
}
$label = $CCAPI->create_full_baseline(%opts);
} else { # incremental labelling
# determine change set - all current checkouts
my ($stat,$out,$err) = cleartool(qw(lsco -avob -cview -short));
if($stat) {
log_both("ERROR: Cannot list current checkouts:\n$out$err\n");
_end_time();
return(0,'');
}
chomp $out;
my @checkouts = reverse sort split(/[\r\n]+/, $out);
$text = "incrementally";
my %opts = (BRANCH => $IntegrationBranch,
COMMENT => "integrated branch(es) @$branches_ref", ADDITIONAL_VOBS => \@AVOBs,
ELEMENTS => \@checkouts);
if($opt{baseline}) {
$opts{LABEL} = $opt{baseline};
}
$label = $CCAPI->create_incremental_baseline(%opts);
}
if($label) {
log_both("INFO: $text labeled baseline '$label' on integration branch '$IntegrationBranch' integrated branches '@$branches_ref' in all VOBs.\n\n");
} else {
log_both("INFO: Branch(es) '@$branches_ref' integrated in all VOBs.\n");
log_both("ERROR: There were problems while $text labelling, please check with integrator.\n");
}
_end_time();
# recommend this baseline
if($label) {
$CCAPI->set_recommended_baseline(BRANCH => $IntegrationBranch, LABEL => $label);
return(1, $label);
} else {
return(0, $label);
}
}
# set state to integrated, lock task branch type
sub _close_branch # REVIEWED
{
my ($branch) = @_;
_start_time("close branch");
my $status = 1;
foreach my $vob ($CCAPI->get_vobs) {
# set the branch status
# The branch type may already be locked - from a previous integration
my ($stat,$out,$err) = cleartool(qw(lslock), "brtype:$branch\@$vob");
if(!$stat && $out !~ /^\s*$/s) {
# there is a lock - unlock it!
($stat,$out,$err) = cleartool(qw(unlock), "brtype:$branch\@$vob");
if($stat) {
log_integ("ERROR: Cannot unlock branch '$branch' in VOB '$vob':\n$out$err\n");
} else {
log_integ("INFO: Unlocked branch '$branch' in VOB '$vob'.\n");
}
}
# TODO if the development continued on the branch, set this back to empty or in_work
($stat,$out,$err) = cleartool(qw(mkattr -replace TASK_STATE "integrated"), "brtype:$branch\@$vob");
if($stat) {
$status = 0;
log_integ("ERROR: Cannot set branch '$branch' status to 'integrated' in VOB '$vob':\n$out$err\n");
} else {
log_integ("INFO: Branch '$branch' status set to 'integrated' in VOB '$vob'.\n");
}
# lock task branch, except for admin (to allow transfer of mastership and hyperlink creation)
if($opt{'lock'}) {
# to be processed at the end of the script
$CCAPI->set_lock_status(BRANCH => $branch, OBJECT => "brtype:$branch\@$vob",
OPTIONS => [ '-obsolete', -c => "Task branch integrated, no more development" ]);
}
} # loop VOBs
_end_time();
return $status;
}
# set hyperlink brtype->lbtype, set integ branch attribute, lock baseline label
sub _close_baseline # REVIEWED
{
my ($baseline, @branches) = @_;
_start_time("close baseline");
my $status = 1;
# create a hyperlink to mark in which baseline(s) the branch(es) was/were integrated
foreach my $branch (@branches) {
unless($CCAPI->create_hyperlink(TYPE => 'DELIVERED_TO', FROM => "brtype:$branch",
TO => "lbtype:$baseline")) {
$status = 0;
}
# lock the baseline label
unless($CCAPI->lock_type(TYPE => "lbtype:$baseline",
OPTIONS => [ -nusers => $IntegUser ],
COMMENT => "no changes on baseline labels")) {
$status = 0;
}
} # loop branches
_end_time();
return $status;
}
sub _get_email # REVIEWED
{
my ($user) = @_;
my $server = 'ldapserver1';
my $basedn = 'ou=People,ou=Urania,ou=Eng,dc=intel,dc=com';
if(open(IN, "</etc/ldap.defaults")) {
while(<IN>) {
if(/^\s*basedn:\s*(.*?)\s*$/s) {
$basedn = $1;
}
elsif(/^\s*server:\s*(\S+)/) {
$server = $1;
}
}
close(IN);
}
elsif(open(IN, "</etc/ldap.conf")) {
while(<IN>) {
if(/^\s*base\s+(.*?)\s*$/s) {
$basedn = $1;
}
elsif(/^\s*host\s+(\S+)/) {
$server = $1;
}
}
close(IN);
}
my $ldap = Net::LDAP->new($server);
unless($ldap) {
log_integ("ERROR: Cannot create LDAP object to determine email address.\n");
return;
}
my $emsg = $ldap->bind();
if(!$emsg || $emsg->is_error) {
log_integ("Cannot bind to LDAP server: ".($emsg ? $emsg->error : '<unspecified>')."\n");
return;
}
# do the search
my $ldapmatch = $ldap->search(
base => "ou=People,$basedn",
scope => "sub",
filter => "(uid=$user)",
attrs => "mail"
);
# extract array (!) of found entries
my @entries = $ldapmatch->entries;
$ldap->unbind();
return unless @entries;
# assume we have only one entry (might be several, but not here, cause uid is primary key)
# fetch attribute (might be multivalued attribute, thus it's an array, too)
my @values = $entries[0]->get_value("mail");
# first (hopefully only) mail attribute
my $email = $values[0];
# hack: try the infineon.com instead of the imc.local address
$email =~ s/\@imc\.local$/\@infineon.com/;
return $email;
}
sub _send_developer_mail # REVIEWED
{
my ($integ_branch, $ok, $logfiles_ref, @branches) = @_;
_start_time("developer mail");
# get branch email addresses
my %owners;
foreach my $branch (@branches) {
foreach my $vob ($CCAPI->get_vobs) {
my ($stat,$out,$err) = cleartool(qw(lstype -fmt %u\n), "brtype:$branch\@$vob");
if($stat) {
log_integ("ERROR: Cannot list branch '$branch' in VOB '$vob':\n$out$err");
next;
}
chomp $out;
my $email = _get_email($out);
$owners{$email}++ if $email;
}
}
unless(keys %owners) {
_end_time();
log_integ("ERROR: Cannot determine owner email addresses for branch(es) '@branches'.\n");
return;
}
my @recipients = keys %owners;
my @integrators = _get_integrators($integ_branch);
unshift(@DeveloperMessage, "Information for developer of branch(es): @branches\n\n");
log_integ("INFO: Sending mail on branch(es) '@branches' to @recipients\n");
_send_email(
$integrators[0],
\@recipients,
"[scintegrate] ".($ok ? 'SUCCESS' : 'FAILURE'). " integration of branch(es): @branches",
\@DeveloperMessage,
$logfiles_ref
);
_end_time();
# clear message buffer - other branches may follow
@DeveloperMessage = ();
}
my $Integrators;
sub _get_integrators # REVIEWED
{
my ($integ_branch) = @_;
if($Integrators) {
return @$Integrators;
}
# get email addresses
my @integrators;
my %seen;
if($opt{integrator}) {
@integrators = grep(!$seen{$_}++, split(/[,;\s]+/, $opt{integrator}));
} else {
foreach my $vob ($CCAPI->get_vobs) {
%seen = ();
my ($stat,$out,$err) = cleartool('desc', -fmt => '%[INTEGRATOR_EMAIL]a\\n', "brtype:$integ_branch\@$vob");
if($stat) {
log_integ("WARNING: Cannot get INTEGRATOR_EMAIL attribute for branch '$integ_branch' in VOB '$vob':\n$out$err");
}
elsif($out =~ /\bINTEGRATOR_EMAIL="([^"]+)"/) {
@integrators = grep(!$seen{$_}++, @integrators, split(/[,;\s]+/, $1));
}
}
}
unless(@integrators) {
# use the current user's email
my $address = _get_email($IntegUser);
if($address) {
push(@integrators, $address);
} else {
warn "ERROR: Cannot determine email address for current user.\n";
}
}
unless(@integrators) {
warn "ERROR: No integrator email addresses defined.\n";
}
$Integrators = \@integrators;
return @integrators;
}
sub _send_integrator_mail # REVIEWED
{
my ($integ_branch, $branches_ref, $success_ref, $logfiles_ref) = @_;
my @integrators = _get_integrators($integ_branch);
return 0 unless(@integrators);
my $subject;
if($Ok && @$success_ref == @$branches_ref) {
$subject = "[scintegrate] SUCCESS integration of branch(es): @$success_ref";
}
elsif(@$branches_ref) {
$subject = "[scintegrate] FAILURE integration of branch(es): @$branches_ref";
}
else {
$subject = "[scintegrate] FAILURE integration";
}
unshift(@IntegratorMessage, <<"EOT");
Summary for integrator:
Processed branches:
@$branches_ref
Successfully integrated:
@$success_ref
EOT
_send_email($integrators[0], \@integrators, $subject, \@IntegratorMessage,
$logfiles_ref);
}
sub _send_email # REVIEWED
{
my ($from, $to, $subject, $lines_ref, $attachments_ref) = @_;
unless($from) {
warn "ERROR: Cannot determine a valid sender email address.\n";
return;
}
# collect attachments from command line:
my @extra_attachments;
foreach(split(/[,\s]+/, $opt{attach})) {
if(-r) {
push(@extra_attachments, $_);
} else {
log_integ("WARNING: File '$_' to be attached is not readable: $!\n");
}
}
### Create a new multipart message:
my $msg = MIME::Lite->new(
From => $from,
To => $to,
Subject => $subject,
Type => 'multipart/mixed'
);
### Add parts (each "attach" has same arguments as "new"):
$msg->attach(
Type => 'TEXT',
Data => join('', @$lines_ref)
);
foreach my $at (@$attachments_ref, @extra_attachments) {
$msg->attach(
Type => 'AUTO', # we have MIME::Types
Path => $at,
Filename => basename($at),
Disposition => 'attachment'
);
}
### use Net:SMTP to do the sending
$msg->send(smtp => $opt{smtp});
1;
}
sub _cancel_checkouts # REVIEWED
{
my ($checkouts_ref) = @_;
my $status = 1;
my $have_checkouts = 0;
my ($stat,$out,$err) = cleartool(qw(lsco -cview -avob -short));
if($stat) {
log_integ("ERROR: Cannot list checkouts:\n$out$err");
} else {
chomp $out;
# make sure to uncheckout files before their parent directories
my @cos = reverse sort split(/[\r\n]+/, $out);
if($checkouts_ref) {
# try to reject only the current checkouts
my @newlist;
my @overlap;
foreach(@cos) {
if(defined $checkouts_ref->{$_}) {
if($OpenCheckouts{$_}) {
# there is an overlap - we cannot selectively cancel
push(@overlap, $_);
} else {
push(@newlist, $_);
}
}
}
if(@overlap) {
my $msg = join("\n", map { " $_" } @overlap);
log_integ(<<"EOT");
ERROR: Merges already happened on these elements; cannot selectively
cancel the checkouts for current task branch:
$msg
EOT
$status = 0;
} else {
@cos = @newlist; # only cancel these
}
} # have checkouts_ref
log_integ("INFO: Cancelling ".scalar(@cos)." checkout(s)...\n") if @cos;
while(my @list = splice(@cos, 0, $opt{batch})) {
$have_checkouts++;
($stat,$out,$err) = cleartool(qw(unco -rm), @list);
if($stat) {
log_integ("ERROR: Problems while cancelling checkouts:\n$out$err");
}
} # loop cos
} # list checkouts
log_integ("INFO: Done cancelling checkouts.\n") if $have_checkouts;
return $status;
}
sub _start_time
{
$_time_task = shift;
$_ts = time();
my $lts = localtime $_ts;
log_integ("\t--- Start task time measurement for $_time_task at $lts ---\n");
}
sub _end_time
{
my $diff = time() - $_ts;
log_integ(sprintf("\t--- End task time measurement for $_time_task. Total running time: %02d:%02d:%02d ---\n\n", int($diff / 3600), int(($diff % 3600) / 60), int($diff % 60)));
return $diff;
}
sub log_dev
{
push(@DeveloperMessage, @_);
print @_;
}
sub log_integ
{
push(@IntegratorMessage, @_);
print @_;
}
sub log_both
{
push(@DeveloperMessage, @_);
push(@IntegratorMessage, @_);
print @_;
}
# things to do whenever the script exits
END {
# remove any locks at the end of the script, but only in parent process
if($$ == $PID) {
# TODO keep lock in case of fatal errors?
$CCAPI->unlock_integration_branches();
# set/remove/restore locks on branch types
$CCAPI->restore_locks_on_all_branches();
# return the mastership of task branches
# - this is only possible if the task branch is not locked obsolete
unless($opt{'lock'}) {
$CCAPI->return_all_branch_masterships();
}
# error exit status, cancel checkouts
if($?) {
_cancel_checkouts() if $HaveCheckouts;
} else {
# ok status - clean up findmerge logs
unlink(glob('./findmerge.log.*'));
}
# send summary message to integrator
_send_integrator_mail($IntegrationBranch, \@BranchQueue,
\@success_integrations, \@AllLogfiles)
if(@IntegratorMessage);
}
}
1;
__END__
=head1 NAME
scintegrate - integrate a task branch in smartCM
=head1 SYNOPSIS
B<scintegrate> S<{ B<-help> | B<-man> | B<-setup> | B<-show> | B<-check> }>
B<scintegrate>
S<[ B<-acs> I<acs-defintion> ]>
S<[ B<-attach> I<path,path2,...> ]>
S<[ B<-baseline> I<label-name> ]>
S<[ B<-batch> I<num> ]>
S<[ B<-freeze> I<timestamp> ]>
S<[ B<-full> | B<-nofull> ]>
S<[ B<-ibranch> I<integration-branch> ]>
S<[ B<-integrator> I<email,...> ]>
S<[ B<-lock> | B<-nolock> ]>
S<[ B<-nobaseline> ]>
S<[ B<-onebaseline> ]>
S<[ B<-oneregression> ]>
S<[ B<-regression> I<script> ]>
S<[ B<-server> I<host:port> ]>
S<[ B<-teelog> ]>
S<[ B<-timeout> I<minutes> ]>
S<[ B<-touch> ]>
S<[ B<-vobs> I<vob-tag,vob-tag2,...> ]>
S<[ B<-ziplog> ]>
S<[ I<task-branch> ... ]>
=head1 DESCRIPTION
B<scintegrate> implements the automated integration process in a continuous
integration development scenario. The script can be executed interactively
as well as automatically e.g. from a cron job or a continuous integration
server. It uses the smartCM ACS (abstract config specs) definitions.
The script must be executed on UNIX from within the project environment, e.g.
like this:
inway5 <project> <subproject> <workspace> -exec 'scintegrate ...'
At script startup, the workspace should have set a config spec corresponding
to the integration branch, in order to pick up the correct version of this
script and its requried modules.
No checkouts are allowed in the current view when starting this script to
avoid an inconsistent state of the new baseline. The script has to be run
by the project administrator (VOB owner), because of some administrative
commands executed in the VOBs.
The script automates the following steps:
=over 4
=item *
"Lock" the integration branch to avoid two concurrent integrations from
starting at the same time.
=item *
Iterate over all given task branches or the list of branches in state "ready"
(for integration).
=item *
If applicable, get the mastership of the integration and task branch types.
=item *
Configure the integration view with the ACS defined on the task branch (but
without the mkbranch rules for the task branch itself).
=item *
Check out the modified elements (this may again imply request-for-mastership).
ClearCase will automatically checkout the latest version on the integration
branch.
=item *
Merge all changes from the task branch. If there are merge conflicts that
cannot be automatically resolved, cancel the merges and checkouts and reject
this task branch. The task branch gets annotated with state "rejected".
=item *
If a regression script was specified, run the script. If it returns non-zero
exit status (= failure), then the checkouts and merges are cancelled and the
branch is rejected. Otherwise continue.
=item *
Check in all successful merges and remember the change set.
=item *
Create a new baseline label type and apply the label (default: incremental
labeling of the change set only).
=item *
Lock the label and the task branch type (obsolete), annotate the task branch
with the status "integrated", and create a hyperlink to the integration
baseline label type.
=item *
Send the results by email to the task branch owner and the integrator,
including the log file of the regression script.
=item *
If a new baseline was created, it is recommended on the integration branch
type by attaching the attribute C<RECOMMENDED_LABEL> with the name of the new
baseline. Finally the lock on the integration branch is released.
=back
=head1 ARGUMENTS
If no arguments are specified, the script will determine integratable
task branch candidates (those that are direct children of the
integration branch) by examining the attribute TASK_STATE: if it is set
to "ready" (meaning ready for integration), the branch will be
considered for integration.
You can override this by specifying a list of task branches to be integrated
on the command line. All given task branches will be integrated into the
same integration branch, regardless of their state.
=head1 OPTIONS
=over 4
=item B<-help>
Show a brief help message.
=item B<-man>
Show this detailed manual.
=item B<-setup>
The project administrator has to execute B<scintegrate -setup> once in the
project to create the required attribute types in all VOBs.
=item B<-show>
Only show the list of branches marked as ready for integration.
=item B<-check>
Like show, but the exit status is the number of task branches ready for
integration (0: no task branch to integrate).
=item B<-acs> I<acs-selector>
Use the given ACS selector for running the integration in. Default is to
use the same ACS selector as defined on the task branch. The syntax is:
[user@]project:acsname[#revision]
The corresponding ACS is retrieved from the ACS server and used to
configure the integration view with.
=item B<-attach> I<path,path2,...>
Attach the given list of paths (comma or space separated, be sure to quote
properly when using spaces) to the emails sent to developer and integrator. The
files should have a unique basename each. If the path does not exist at the
time when the email is sent, only a warning message is raised. Note that the
SMTP server may have restrictions concerning the maximum size of the messages.
=item B<-baseline> I<label-name>
Specify a custom baseline label name. If not specified, B<scintegrate>
will use the baseline counter (attribute C<BASELINE_COUNTER> on the
integration branch type) to determine the new baseline name, by
incrementing the number suffix of the counter by 1.
=item B<-freeze> I<timestamp>
The ACS may contain the C</freeze> modifier in the selector; this indicates (only in
connection with C<LATEST>) that the latest version from the given branch should be
shown, but only until a certain time stamp. When B<scintegrate> is called and
generates the integration config spec, it will use the current time as the cutoff
timestamp. With this option you can specify another time stamp. Use the same syntax as
documented in the B<config_spec> manual page of ClearCase:
<mm>-<mon>-<year>.<hh>:<mm>[:ss] e.g. 01-Aug-2011.13:30
=item B<-ibranch> I<integration-branch>
The default integration branch is "main", but this option allows to specify
a different branch for integration. This way a family development approach
can be implemented, where e.g. a new spin of a product is integrated on a
separate branch, which is rooted at the tape-out label of the predecessor
product. If not specified, B<scintegrate> will read the integration
branch from the L</INTEGRATION_BRANCH> environment variable.
=item B<-integrator> I<email,...>
This option allows to specify a list of email addresses to which to
send the email about the integration results. If not specified, the
integrator is determined from the attribute INTEGRATOR_EMAIL attached
to the integration branch type.
=item B<-lock> | B<-nolock>
After a successful integration, by default the task branch is locked
obsolete to avoid any further changes on the integrated task branch and
keep the version tree lean. This can be disabled with the B<-nolock>
option.
=item B<-nobaseline>
Suppress the creation of a new baseline after integration. The merged
versions are nevertheless checked in on the integration branch.
=item B<-onebaseline>
Normally, this script creates a baseline I<for each> successfully
integrated task branch. This option allows to only create one baseline
I<for all> successfully integrated task branches. The rest of the
process (merge, run regression, check in) remains the same.
=item B<-oneregression>
Normally, this script runs the given regression script I<for each> task
branch merged. With this option, the script tries to merge as many task
branches as possible (without conflicts), then runs the regression, and
if OK, completes the integration for all merged task branches. If the
regression fails, all merges are cancelled and information is sent to
all involved task branch owners - since it is not possible to uniquely
identify the "culprit" (not if only one single task branch was merged).
In case there are merge conflicts, the script tries to selectively
cancel the checkouts related to the failing task branch. If possible,
this branch is rejected and the script continues with the next task
branch. Otherwise all checkouts are canceled and the entire integration
is aborted. The integration should then be restarted with a selective
list of task branches.
=item B<-regression> I<script>
The script to run the regression or smoke test on the merged changes.
This decides whether the changes will be accepted or not. The script's
output is captured into a log file and has to return a zero exit status
in case of success, non-zero in case of failure.
If not specified, it is read from the ACS (keyword: C<on-integration>).
=item B<-server> I<host:port>
The ACS server I<host:port> combination. If not specified, it is read
from the environment L</ACS_SERVER>.
=item B<-stmp> I<hostname>
The name of the mail server host to use to send the emails to the
developer and integrator. Default is C<smtp.imu.imc.local>.
=item B<-teelog>
With this option the output of the regression script is printed while
running by scintegrate, and also captured in the log file. Default is
to redirect the output only to the log file to keep the output of
scintegrate short.
=item B<-timeout> I<minutes>
The default value is 5 minutes; this is the time the script waits for
mastership to arrive. Mastership transfers are required to be able to
e.g. check out on the integration branch locally.
=item B<-touch>
Before merging, "touch" the (checked-out) merge destination in order to
get rid of any derived objects. This may be necessary when derived
object versions are checked in on task branches. Future versions of
scintegrate may actually preserve the derived objects during a copy-merge
operation.
=item B<-vobs> I<tag1,tag2,...>
By default, the list of VOBs is determined from the standard ClearCase
environment variable C<CLEARCASE_AVOBS> and the ACS definition.
This option allows you to specify a (shorter) list of VOBs to process.
=item B<-ziplog>
Attach the regression log file compressed as F<*.zip>. Default is to
attach the file as plain text (F<*.log>).
=back
=head1 ENVIRONMENT
=over 4
=item INTEGRATION_BRANCH
This environment variable may be set to contain the default
integration branch. It has the same effect as the B<-ibranch>
option.
=item ACS_SERVER
The ACS server host:port to use. Default value is
C<lnzll03d.lnz.infineon.com:3100>.
=back
=head1 INTERNALS
The script uses the following attributes (attached to the branch types in
each VOB) to manage the properties of a branch:
=over 4
=item INTEGRATING
This attribute is used to "lock" the integration branch type, to avoid two
integration processes running simultaneously.
=item INTEGRATOR_EMAIL
This attribute, attached to the integration branch type, contains the email
address(es) of the integrators to be notified of the integration result.
=item TASK_STATE
This attribute may take the values C<open>, C<ready>, C<rejected>, or
C<integrated>. The B<ccbranch> script sets the C<TASK_STATE> to
C<ready> when the script is run with the B<-deliver> option. The
B<scintegrate> script sets this attribute to "rejected" if the
integration failed (merge conflicts or failed regression), or to
"integrated" in case of integration success.
=item INTEGRATION_BRANCH (attribute)
This attribute is attached to the task branch type to store the default
integration branch with the task branch. Furthermore it is attached
to the baseline label types created by B<scintegrate>.
This attribute allows to identify all child branches of an integration
branch, as well as the default rebase labels.
=item DELIVERED_TO
This hyperlink type is used to identify to which baseline label a task
branch was integrated to. It points (in each VOB) from the task branch
type to the baseline label type, and is thus easy to identify. The
individual merged versions can then be determined with a find command.
=item ACS
The ACS definition string; this is attached to the task branch type to
identify on which ACS the task branch was configured.
=item BASELINE_COUNTER
The name for new baselines (incremental or full) to be created on the
integration branch. It has to have a numeric suffix which will be
automatically incremented whenever a new baseline is created with
B<scintegrate> on the integration branch.
=item BRANCH_ROOT_LABEL
This attribute is required for integration branches and references a
fully labeled baseline which is the foundation of the integration
branch.
=item RECOMMENDED_LABEL
Describes the most recent successfully integrated baseline on the
integration branch. Used to determine the "HEAD" baseline on the branch.
=back
=head1 SEE ALSO
L<ccbranch>, L<ccco>, L<ClearCase::CtCmd>, L<ClearCase::MtCmd>
=cut