Mercurial > pub > buggler

changeset 10:14a966369278

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
working version of exporting bugs from bugzilla in tj3 format (without bookings)
author cin
date Mon, 07 Sep 2015 01:37:11 +0300
parents cc7244ab1b9f
children 4eb9fdf4efa9
files bug-list.xsl lib/Benzin/Bugzilla/Bug.pm lib/Benzin/Bugzilla/XmlRpcClient.pm lib/Benzin/Bugzilla/XmlRpcDeserializer.pm lib/Benzin/Bugzilla/XmlSerializer.pm lib/Benzin/Bugzilla/XmlWriter.pm translate.pl
diffstat 7 files changed, 245 insertions(+), 156 deletions(-) [+]
line wrap: on
line diff
--- a/bug-list.xsl	Sat Sep 05 22:01:12 2015 +0300
+++ b/bug-list.xsl	Mon Sep 07 01:37:11 2015 +0300
@@ -22,9 +22,9 @@
 
 	<xsl:variable name="bugs" select="/bugzilla/bug" />
 
-	<xsl:key name="bugid" match="/bugzilla/bug" use="string(bug_id)" />
+	<xsl:key name="bugid" match="/bugzilla/bug" use="string(id)" />
 
-	<xsl:variable name="roots" select="$bugs[not(blocked[key('bugid',.)])]" />
+	<xsl:variable name="roots" select="$bugs[not(blocks[key('bugid',.)])]" />
 
 	<!-- BUILD BUG TREE -->
 	<xsl:variable name="tree">
@@ -33,11 +33,11 @@
 
 	<xsl:template match="bug" mode="tree">
 		<xsl:element name="bug">
-			<xsl:attribute name="id"><xsl:value-of select="bug_id" /></xsl:attribute>
+			<xsl:attribute name="id"><xsl:value-of select="id" /></xsl:attribute>
 			<xsl:if test="component = 'product' or not(number(estimated_time))">
 				<xsl:attribute name="group"><xsl:value-of select="boolean(1)" /></xsl:attribute>
 			</xsl:if>
-			<xsl:for-each select="dependson">
+			<xsl:for-each select="depends_on">
 				<xsl:apply-templates mode="tree" select="key('bugid', .)" />
 			</xsl:for-each>
 		</xsl:element>
@@ -49,7 +49,7 @@
 	</xsl:variable>
 
 	<xsl:template match="bug" mode="relations">
-		<xsl:variable name="bugid" select="string(bug_id)" />
+		<xsl:variable name="bugid" select="string(id)" />
 		<bug id="{$bugid}">
 			<!-- calculate all palaces where the same node appears -->
 			<xsl:apply-templates select="exsl:node-set($tree)//bug[@id = $bugid]"
@@ -106,20 +106,20 @@
 
 	<!-- applied in context of the bugzilla document -->
 	<xsl:template match="bug" mode="structure">
-		<xsl:variable name="id" select="string(bug_id)" />
+		<xsl:variable name="id" select="string(id)" />
 		<xsl:variable name="self" select="." />
 		<xsl:variable name="children"
 			select="exsl:node-set($parents)/bug[@parent = $id]" />
 		<xsl:element name="bug">
 			<xsl:attribute name="id"><xsl:value-of select="$id" /></xsl:attribute>
-			<xsl:attribute name="desc"><xsl:value-of select="short_desc" /></xsl:attribute>
+			<xsl:attribute name="desc"><xsl:value-of select="summary" /></xsl:attribute>
 
 			<xsl:if test="$children">
 				<xsl:attribute name="group"><xsl:value-of select="boolean($children)" /></xsl:attribute>
 
 				<xsl:for-each select="$children">
 					<xsl:variable name="child" select="@id" />
-					<xsl:apply-templates select="$bugs[bug_id = $child]"
+					<xsl:apply-templates select="$bugs[id = $child]"
 						mode="structure" />
 				</xsl:for-each>
 
@@ -128,7 +128,7 @@
 			<!-- filter out dependencies -->
 			<!-- exclude children, and missing bugs -->
 			<xsl:variable name="dependencies"
-				select="dependson[not(text() = exsl:node-set($parents)/bug/parent[@id=$id]/@bugid)][key('bugid', .)]" />
+				select="depends_on[not(text() = exsl:node-set($parents)/bug/parent[@id=$id]/@bugid)][key('bugid', .)]" />
 			<xsl:for-each select="$dependencies">
 				<dependency id="{.}" />
 			</xsl:for-each>
@@ -147,7 +147,7 @@
 		<xsl:param name="indent" select="0" />
 		<xsl:param name="chargeset" />
 		<xsl:variable name="id" select="@id" />
-		<xsl:variable name="details" select="$bugs[bug_id = $id]" />
+		<xsl:variable name="details" select="$bugs[id = $id]" />
 		<xsl:variable name="hasTime"
 			select="number($details/estimated_time) or number($details/actual_time) or number($details/remaining_time)" />
 
--- a/lib/Benzin/Bugzilla/Bug.pm	Sat Sep 05 22:01:12 2015 +0300
+++ b/lib/Benzin/Bugzilla/Bug.pm	Mon Sep 07 01:37:11 2015 +0300
@@ -46,22 +46,10 @@
 use constant { BUG_FIELDS => \@fields };
 
 use IMPL::declare {
-	require => {
-		Strptime => 'DateTime::Format::Strptime'
-	},
-	base => [
-		'IMPL::Object::Fields' => undef
-	]
+	base    => [ 'IMPL::Object::Fields' => undef ]
 };
-
 use fields @fields;
 
-my $dtparser = Strptime->new(
-	pattern => '%Y%m%dT%H:%M:%S',
-	time_zone => 'UTC',
-	on_error => 'croak'
-);
-
 sub CTOR {
 	my SELF $this = shift;
 	my $data = shift;
@@ -71,7 +59,7 @@
 
 # returns {
 #	reports => [
-#		{ who => email:string, when => report-date-time:DateTime, work_time => hours:double }
+#		{ who => email:string, start => work-start-date-time:DateTime, end => report-date-time:DateTime, work_time => hours:double }
 #	],
 #   actual => hours
 #	remaining => hours
@@ -79,16 +67,13 @@
 sub GetTimeReports {
 	my SELF $this = shift;
 	my $resolution = shift || 0.25;
-	
-	warn "Processing: $this->{id}";
 
 	my @bookings;
 	my $actual = 0;
 
 	for my $history ( @{ $this->{history} || [] } ) {
-		my $who     = $history->{who};
-		warn $history->{when};
-		my $when    = $dtparser->parse_datetime( $history->{when} );
+		my $who = $history->{who};
+		my $when    = $history->{when};
 		my $changes = $history->{changes};
 
 		for my $change ( @{ $changes || [] } ) {
@@ -101,10 +86,11 @@
 					if ($dt) {
 						push @bookings,
 						  {
-							who       => $who,
+							end       => $who,
 							when      => $when->iso8601(),
 							work_time => $dt,
-							start     => $when->clone()->subtract( hours => $dt )->iso8601()
+							start => $when->clone()->subtract( hours => $dt )
+							  ->iso8601()
 						  };
 						$actual += $dt;
 					}
@@ -125,4 +111,4 @@
 	return $resolution ? ceil( $value / $resolution ) * $resolution : $value;
 }
 
-1;
\ No newline at end of file
+1;
--- a/lib/Benzin/Bugzilla/XmlRpcClient.pm	Sat Sep 05 22:01:12 2015 +0300
+++ b/lib/Benzin/Bugzilla/XmlRpcClient.pm	Mon Sep 07 01:37:11 2015 +0300
@@ -1,29 +1,20 @@
 package Benzin::Bugzilla::XmlRpcClient;
 use strict;
 
-use LWP::UserAgent;
-use XMLRPC::Lite;
 use YAML::XS qw(Dump);
+use URI;
 
 use IMPL::declare {
 	require => {
-		Bug        => 'Benzin::Bugzilla::Bug',
-		BugComment => 'Benzin::Bugzilla::BugComment'
+		Bug          => 'Benzin::Bugzilla::Bug',
+		BugComment   => 'Benzin::Bugzilla::BugComment',
+		Deserializer => 'Benzin::Bugzilla::XmlRpcDeserializer',
+		XmlRpc       => 'XMLRPC::Lite'
 	},
 	base => { 'IMPL::Object::Fields' => undef }
 };
 
-use fields qw(url apikey);
-
-sub new {
-	my $class = shift;
-	$class = ref $class || $class;
-
-	my $inst = fields::new($class);
-	$inst->CTOR(@_);
-
-	return $inst;
-}
+use fields qw(url apikey _client);
 
 sub CTOR {
 	my SELF $this = shift;
@@ -31,6 +22,13 @@
 
 	$this->{url} = $params{url} or die "An url is required";
 	$this->{apikey} = $params{apikey} if $params{apikey};
+	
+	#create slightly modified client to parse datetime values
+	my $url = URI->new_abs( 'xmlrpc.cgi', $this->{url} );
+	
+	$this->{_client} =
+	  XmlRpc->new( deserializer => Deserializer->new() )->proxy($url);
+
 }
 
 sub GetBugs {
@@ -42,7 +40,35 @@
 	];
 }
 
-sub PopulateBugsWithComments {
+sub GetBugsHierarchy {
+	my SELF $this = shift;
+	my $args = shift || {};
+	
+	my @queue = @{$args->{ids}};
+	my %visited;
+	
+	my @fetched;
+
+	while (@queue) {
+		@queue = grep not( $visited{$_}++ ), @queue;
+
+		last unless @queue;
+
+		my $bugs = $this->GetBugs( { ids => \@queue } );
+		@queue = ();
+
+		foreach my Bug $bug (@$bugs) {
+
+			push @queue, @{ $bug->{depends_on} }
+			  if ( $bug->{depends_on} );
+			push @fetched, $bug;
+		}
+	}
+	
+	return \@fetched;
+}
+
+sub PopulateBugsComments {
 	my SELF $this = shift;
 	my $bugs = shift || [];
 
@@ -68,11 +94,12 @@
 	if ( keys %bugs ) {
 
 		my $resp =
-		  $this->_CallService( 'Bug.history', { ids => [ keys %bugs ] } )->{bugs};
+		  $this->_CallService( 'Bug.history', { ids => [ keys %bugs ] } )
+		  ->{bugs};
 
 		for my $data (@$resp) {
-			my Bug $bug = $bugs{$data->{id}};
-			
+			my Bug $bug = $bugs{ $data->{id} };
+
 			$bug->{history} = $data->{history};
 		}
 	}
@@ -87,9 +114,9 @@
 	$params ||= {};
 
 	$params->{api_key} = $this->{apikey};
-	my $url = URI->new_abs( 'xmlrpc.cgi', $this->{url} );
+	
 
-	my $result = XMLRPC::Lite->proxy($url)->call( $method, $params );
+	my $result = $this->{_client}->call( $method, $params );
 
 	die $result->fault if $result->fault;
 	return $result->result;
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/lib/Benzin/Bugzilla/XmlRpcDeserializer.pm	Mon Sep 07 01:37:11 2015 +0300
@@ -0,0 +1,51 @@
+package Benzin::Bugzilla::XmlRpcDeserializer;
+use strict;
+use mro;
+use parent qw(-norequire XMLRPC::Deserializer);
+use IMPL::require { Strptime => 'DateTime::Format::Strptime' };
+
+my $xmlRpcDateFormat = Strptime->new(
+	pattern   => '%Y%m%dT%H:%M:%S',
+	time_zone => 'UTC',
+	on_error  => 'croak'
+);
+
+sub decode_value {
+	my $this = shift;
+	my ( $name, $attrs, $children, $value ) = @{ $_[0] };
+
+	if ( $name eq 'dateTime.iso8601' ) {
+		return $xmlRpcDateFormat->parse_datetime($value);
+	}
+	else {
+		$this->next::method(@_);
+	}
+
+}
+
+1;
+
+__END__
+
+=pod
+
+=head1 NAME
+
+C<Benzin::Bugzilla::XmlRpcDeserializer> - data deserializer for L<XMLRPC::Lite>.
+
+=head1 SYNOPSIS
+
+=begin code perl
+
+# override the default deserializer
+my $client = XMLRPC::Lite->new(
+	deserializer => Benzin::Bugzilla::XmlRpcDeserializer->new()
+);
+
+=end code perl
+
+=head1 DESCRIPTION
+
+Slightly tuned C<XMLRPC::Deserializer> to parse C<dateTime.iso8601> values to L<DateTime> objects.
+
+=cut
--- a/lib/Benzin/Bugzilla/XmlSerializer.pm	Sat Sep 05 22:01:12 2015 +0300
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,67 +0,0 @@
-package Benzin::Bugzilla::XmlSerializer;
-
-use IMPL::declare {
-	require => {
-		XMLWriter => 'XML::Writer'
-	},
-	base => [
-		'IMPL::Object::Fields' => undef
-	]
-};
-
-use fields qw(_writer);
-
-sub CTOR {
-	my SELF $this = shift;
-	
-	$this->{_writer} = XMLWriter->new(@_);
-}
-
-sub WriteBugList {
-	my SELF $this = shift;
-	my $bugs = shift || [];
-	my $writer = $this->{_writer};
-	
-	
-	$writer->xmlDecl("UTF-8");
-	$writer->startTag("bugzilla");
-	
-	$writer->startElement("bugs");
-	
-	$this->WriteBug($_) foreach @$bugs;
-	
-	$writer->endTag();
-	$writer->endTag();
-	
-}
-
-sub WriteBug {
-	my SELF $this = shift;
-	my $bug       = shift;
-	my $writer    = $this->{_writer};
-
-	$writer->startTag("bug");
-	foreach my $field ( @{ Bug->BUG_FIELDS } ) {
-		next unless $bug->{$field};
-		$this->WriteElement( $field, $bug->{$field} );
-	}
-	$writer->endTag();
-}
-
-sub WriteElement {
-	my SELF $this = shift;
-	my ( $name, $data ) = @_;
-	my $writer = $this->{_writer};
-
-	my @values =
-	     ref($data)
-	  && ref($data) eq 'ARRAY'
-	  ? @{$data}
-	  : $data;
-
-	foreach my $v (@values) {
-		$writer->dataElement( $name, $v );
-	}
-}
-
-1;
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/lib/Benzin/Bugzilla/XmlWriter.pm	Mon Sep 07 01:37:11 2015 +0300
@@ -0,0 +1,110 @@
+package Benzin::Bugzilla::XmlWriter;
+
+our %Transform = (
+	'Benzin::Bugzilla::Bug' => 'WriteBug',
+	'Benzin::Bugzilla::BugComment' => 'WriteBugComment',
+	'HASH' => 'WriteHash',
+	-plain => 'WriteValue',
+	-default => 'WriteValue'
+);
+
+use IMPL::Const qw(:prop);
+use IMPL::declare {
+	require => {
+		XMLWriter => 'XML::Writer',
+		Bug => 'Benzin::Bugzilla::Bug',
+		BugComment => 'Benzin::Bugzilla::BugComment'
+	},
+	base => [
+		'IMPL::Transform' => sub { %Transform }
+	],
+	props => [
+		_writer => PROP_RW
+	]
+};
+
+use fields qw(_writer);
+
+sub CTOR {
+	my SELF $this = shift;
+	
+	$this->_writer(XMLWriter->new(@_));
+}
+
+sub WriteBugList {
+	my SELF $this = shift;
+	my $bugs = shift || [];
+	my $writer = $this->_writer;
+	
+	
+	$writer->xmlDecl("UTF-8");
+	$writer->startTag("bugzilla");
+	
+	
+	foreach my $bug (@$bugs) {
+		$writer->startTag("bug");
+		$this->WriteBug($bug);
+		$writer->endTag();
+	}
+	
+	$writer->endTag();
+	
+}
+
+sub WriteBug {
+	my SELF $this = shift;
+	my $value       = shift;
+	my $writer    = $this->_writer;
+
+	foreach my $field ( @{ Bug->BUG_FIELDS } ) {
+		next unless $value->{$field};
+		$this->WriteElement( $field, $value->{$field} );
+	}
+}
+
+sub WriteBugComment {
+	my SELF $this = shift;
+	my $value       = shift;
+	my $writer    = $this->_writer;
+
+	foreach my $field ( @{ BugComment->COMMENT_FIELDS } ) {
+		next unless $value->{$field};
+		$this->WriteElement( $field, $value->{$field} );
+	}
+}
+
+sub WriteHash {
+	my SELF $this = shift;
+	my $value = shift;
+	
+	while(my ($k,$v) = each %$value) {
+		$this->WriteElement($k,$v);
+	}
+}
+
+sub WriteElement {
+	my SELF $this = shift;
+	my ( $name, $data ) = @_;
+	my $writer = $this->_writer;
+
+	my @values =
+	     ref($data)
+	  && ref($data) eq 'ARRAY'
+	  ? @{$data}
+	  : $data;
+
+	foreach my $v (@values) {
+		$writer->startTag($name);
+		$this->Transform( $v );
+		$writer->endTag;
+	}
+}
+
+sub WriteValue {
+	my SELF $this = shift;
+	my $value = shift;
+	
+	$this->_writer->characters($value) if defined $value;
+}
+
+1;
--- a/translate.pl	Sat Sep 05 22:01:12 2015 +0300
+++ b/translate.pl	Mon Sep 07 01:37:11 2015 +0300
@@ -1,21 +1,13 @@
 #!/usr/bin/perl -w
 
 use IMPL::require {
-	BzClient => 'Benzin::Bugzilla::XmlRpcClient',
-	Bug      => 'Benzin::Bugzilla::Bug'
+	BzClient  => 'Benzin::Bugzilla::XmlRpcClient',
+	BugWriter => 'Benzin::Bugzilla::XmlWriter'
 };
 use YAML::XS qw(LoadFile Dump);
 use XML::Writer;
 use IPC::Run qw(start finish);
 
-our @ClassPath = qw(
-  /usr/share/java/xalan-j2-serializer.jar
-  /usr/share/java/xalan-j2.jar
-  /usr/share/java/xerces-j2.jar
-  /usr/share/java/xml-commons-resolver.jar
-  .
-);
-
 my $config = LoadFile("config.yaml");
 
 if ( !( $config->{bugzilla}{url} =~ /\/$/ ) ) {
@@ -27,40 +19,30 @@
 	apikey => $config->{bugzilla}{apikey}
 );
 
+my $bugs = $config->{bugzilla}{bugs} or die "No bugs specified";
+$bugs = [$bugs] unless ref $bugs eq 'ARRAY';
+
 local (*HIN);
 
-my $proc = start( [ 'saxon8', '-novw', '-', 'bug-list.xsl' ],
-	'<pipe', \*HIN, '>', \*STDOUT )
-  or die "failed to create pipe: $!";
+my $proc = start(
+	[ 'saxon8', '-novw', '-', 'bug-list.xsl' ],
+	#[ 'cat' ],
+	'<pipe', \*HIN, '>', \*STDOUT
+) or die "failed to create pipe: $!";
+
+binmode *HIN, ":encoding(utf-8)";
+my $writer =
+  BugWriter->new( OUTPUT => *HIN, DATA_INDENT => 2, DATA_MODE => 'on' );
 
 eval {
 	my %visited;
-	my @queue = (283);
-	my @fetched;
 
-	while (@queue) {
-		@queue = grep not( $visited{$_}++ ), @queue;
-
-		last unless @queue;
-
-		print "#Fetching: ", join( ', ', @queue ), "\n";
-
-		my $bugs = $bz->GetBugs( { ids => \@queue } );
-		@queue = ();
+	my $fetched = $bz->GetBugsHierarchy( { ids => [283] } );
 
-		foreach my $bug (@$bugs) {
+	$bz->PopulateBugsComments($fetched);
+	$bz->PopulateBugsHistory($fetched);
 
-			push @queue, @{ $bug->{depends_on} }
-			  if ( $bug->{depends_on} );
-			push @fetched, $bug;
-		}
-	}
-	print Dump( \@fetched );
-
-	$bz->PopulateBugsWithComments( \@fetched );
-	$bz->PopulateBugsHistory( \@fetched );
-
-	print Dump( [ map $_->GetTimeReports(0.25), @fetched ] );
+	$writer->WriteBugList($fetched);
 
 };
 warn Dump($@) and die $@ if $@;