SQOOP-2796: Sqoop2: Add base version of the tool
[sqoop.git] / dev-support / upload-patch.py
1 #!/usr/bin/env python
2 #
3 # Licensed to the Apache Software Foundation (ASF) under one
4 # or more contributor license agreements. See the NOTICE file
5 # distributed with this work for additional information
6 # regarding copyright ownership. The ASF licenses this file
7 # to you under the Apache License, Version 2.0 (the
8 # "License"); you may not use this file except in compliance
9 # with the License. You may obtain a copy of the License at
10 #
11 # http://www.apache.org/licenses/LICENSE-2.0
12 #
13 # Unless required by applicable law or agreed to in writing,
14 # software distributed under the License is distributed on an
15 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 # KIND, either express or implied. See the License for the
17 # specific language governing permissions and limitations
18 # under the License.
19
20 # This script will take your local git changes and upload them as a patch JIRA and review
21 # board. This script has been written to support Sqoop workflow but can work for any project
22 # that uses JIRA and review board.
23 #
24 # This tool depends on reviewboard python APIs, please download them
25 # from here: https://www.reviewboard.org/downloads/rbtools/
26 #
27 #
28 # Future improvement ideas
29 # * When submitting review request open an editor to let user fill in the details?
30 # * Add protection against uploading the same file (patch) twice?
31 # * Migrate all HTTP calls from urllib2 to requests?
32 import sys, os, re, urllib2, base64, subprocess, tempfile, shutil
33 import json
34 import datetime
35 import ConfigParser
36 import requests
37 from optparse import OptionParser
38 from rbtools.api.client import RBClient
39
40 # Resource file location
41 RC_PATH = os.path.expanduser("~/.upload-patch.rc")
42
43 # Default option values
44 DEFAULT_JIRA_URL = 'https://issues.apache.org/jira'
45 DEFAULT_JIRA_RB_LABEL = "Review board"
46 DEFAULT_JIRA_TRANSITION = "Patch Available"
47 DEFAULT_RB_URL = 'https://reviews.apache.org'
48 DEFAULT_RB_REPOSITORY = 'sqoop-sqoop2'
49 DEFAULT_RB_GROUP = 'sqoop'
50 DEFAULT_JIRA_USER = None
51 DEFAULT_JIRA_PASSWORD = None
52 DEFAULT_RB_USER = None
53 DEFAULT_RB_PASSWORD = None
54
55 # Loading resource file that can contain some parameters
56 if os.path.exists(RC_PATH):
57 rc = ConfigParser.RawConfigParser()
58 rc.read(RC_PATH)
59 # And override faults from the rc file
60 DEFAULT_JIRA_USER = rc.get("jira", "username")
61 DEFAULT_JIRA_PASSWORD = rc.get("jira", "password")
62 DEFAULT_RB_USER = rc.get("reviewboard", "username")
63 DEFAULT_RB_PASSWORD = rc.get("reviewboard", "password")
64 print "Loaded JIRA username from resource file: %s" % DEFAULT_JIRA_USER
65 print "Loaded Review board username from resource file: %s" % DEFAULT_RB_USER
66 else:
67 print "Resource file %s not found." % RC_PATH
68
69 # Options
70 parser = OptionParser("Usage: %prog [options]")
71 parser.add_option("--jira", dest="jira", help="JIRA number that this patch is for", metavar="SQOOP-1234")
72 parser.add_option("--jira-url", dest="jira_url", default=DEFAULT_JIRA_URL, help="URL to JIRA instance", metavar="http://jira.com/")
73 parser.add_option("--jira-user", dest="jira_user", default=DEFAULT_JIRA_USER, help="JIRA username", metavar="jarcec")
74 parser.add_option("--jira-transition",dest="jira_transition", default=DEFAULT_JIRA_TRANSITION,help="Name of the transition when uploading patch", metavar="Patch Available")
75 parser.add_option("--jira-password", dest="jira_password", default=DEFAULT_JIRA_PASSWORD, help="JIRA passowrd", metavar="secret")
76 parser.add_option("--jira-rb-label", dest="jira_rb_label", default=DEFAULT_JIRA_RB_LABEL, help="Label to be used in JIRA for the review board link", metavar="Review")
77 parser.add_option("--rb-url", dest="rb_url", default=DEFAULT_RB_URL, help="URL to Review board instance", metavar="http://rb.com/")
78 parser.add_option("--rb-group", dest="rb_group", default=DEFAULT_RB_GROUP, help="Review group for new review entry", metavar="sqoop")
79 parser.add_option("--rb-repository", dest="rb_repository", default=DEFAULT_RB_REPOSITORY, help="Review board's repository", metavar="sqoop2")
80 parser.add_option("--rb-user", dest="rb_user", default=DEFAULT_RB_USER, help="Review board username", metavar="jarcec")
81 parser.add_option("--rb-password", dest="rb_password", default=DEFAULT_RB_PASSWORD, help="Review board passowrd", metavar="secret")
82 parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False, help="Print more debug information while execution")
83
84 # Execute given command on command line
85 def execute(cmd, options):
86 if options.verbose:
87 print "Executing command: %s" % (cmd)
88 return subprocess.call(cmd, shell=True)
89
90 # End program execution with given message and return code
91 def exit(message, ret=1):
92 print "FATAL: %s" % message
93 sys.exit(ret)
94
95 # Convert given number of bytes to human readable one
96 # Source: http://stackoverflow.com/questions/1094841/reusable-library-to-get-human-readable-version-of-file-size
97 def human_readable_size(num, suffix='B'):
98 for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
99 if abs(num) < 1024.0:
100 return "%3.1f%s%s" % (num, unit, suffix)
101 num /= 1024.0
102 return "%.1f%s%s" % (num, 'Yi', suffix)
103
104 # Load given file entirely into memory
105 def get_file_content(filepath):
106 f = open(filepath, mode="r")
107 diff = f.read()
108 f.close()
109 return diff
110
111 # Geneate request to JIRA instance
112 def jira_request(url, options, data, headers):
113 request = urllib2.Request(url, data, headers)
114 if options.verbose:
115 print "JIRA Request: URL = %s, Username = %s, data = %s, headers = %s" % (url, options.jira_user, data, str(headers))
116 if options.jira_user and options.jira_password:
117 base64string = base64.encodestring('%s:%s' % (options.jira_user, options.jira_password)).replace('\n', '')
118 request.add_header("Authorization", "Basic %s" % base64string)
119 return urllib2.urlopen(request)
120
121 # Get response from JIRA in form of JSON and parse the JSON for downstream consumption
122 def jira_json(url, options, data, headers):
123 body = jira_request(url, options, data, headers).read()
124 if options.verbose:
125 print "Response: %s" % body
126 return json.loads(body)
127
128 # General details of JIRA issue
129 def jira_get_issue(options):
130 url = "%s/rest/api/2/issue/%s" % (options.jira_url, options.jira)
131 return jira_json(url, options, None, {})
132
133 # Links associated with the JIRA
134 def jira_get_links(options):
135 url = "%s/rest/api/2/issue/%s/remotelink" % (options.jira_url, options.jira)
136 return jira_json(url, options, None, {})
137
138 # Create new link
139 def jira_post_links(link_url, title, options):
140 url = "%s/rest/api/2/issue/%s/remotelink" % (options.jira_url, options.jira)
141 data = '{"object" : {"url" : "%s", "title" : "%s"}}' % (link_url, title)
142 jira_request(url, options, data, {"Content-Type" : "application/json"})
143
144 # Possible transitions for JIRA
145 def jira_get_transitions(options):
146 url = "%s/rest/api/2/issue/%s/transitions?expand=transititions.fields" % (options.jira_url, options.jira)
147 return jira_json(url, options, None, {})
148
149 # Transition JIRA to give state
150 def jira_post_transitions(transitionId, options):
151 url = "%s/rest/api/2/issue/%s/transitions" % (options.jira_url, options.jira)
152 data = '{"transition" : {"id" : "%s"}}' % transitionId
153 jira_request(url, options, data, {"Content-Type" : "application/json"})
154
155 # Create new attachement
156 def jira_post_attachments(f, options):
157 url = "%s/rest/api/2/issue/%s/attachments" % (options.jira_url, options.jira)
158 files = {'file':open(f)}
159 headers = {"X-Atlassian-Token" : "no-check"}
160 requests.post(url, files=files, headers=headers, auth=(options.jira_user, options.jira_password)).text
161
162 # Parse and validate arguments
163 (options, args) = parser.parse_args()
164 if not options.jira:
165 exit("Missing argument --jira")
166
167 # Main execution
168 patch = "%s.patch" % options.jira
169 execute("git diff HEAD > %s" % patch, options)
170 if not os.path.exists(patch):
171 exit("Can't generate patch locally")
172
173 # Verify size of the patch
174 patchSize = os.path.getsize(patch)
175 if patchSize == 0:
176 exit("Generated empty patch, ending gracefully", 0)
177 else:
178 print "Created patch %s (%s)" % (patch, human_readable_size(patchSize))
179
180 # Retrive link to review board if it exists already
181 reviewBoardUrl = None
182 linksJson = jira_get_links(options)
183 for link in linksJson:
184 if link.get("object").get("title") == options.jira_rb_label:
185 reviewBoardUrl = link.get("object").get("url")
186 break
187 if options.verbose:
188 if reviewBoardUrl:
189 print "Found associated review board: %s" % reviewBoardUrl
190 else:
191 print "No associated review board entry found"
192
193 # Saving details of the JIRA for various use
194 print "Getting details for JIRA %s" % (options.jira)
195 jiraDetails = jira_get_issue(options)
196
197 # Review board handling
198 rbClient = RBClient(options.rb_url, username=options.rb_user, password=options.rb_password)
199 rbRoot = rbClient.get_root()
200
201 # The RB REST API don't have call to return repository by name, only by ID, so one have to
202 # manually go through all the repositories and find the one that matches the corrent name.
203 rbRepoId = -1
204 for repo in rbRoot.get_repositories(max_results=500):
205 if repo.name == options.rb_repository:
206 rbRepoId = repo.id
207 break
208 # Verification that we have found required repository
209 if rbRepoId == -1:
210 exit("Did not found repository '%s' on review board" % options.rb_repository)
211 else:
212 if options.verbose:
213 print "Review board repository %s has id %s" % (options.rb_repository, rbRepoId)
214
215 # If review doesn't exists we need to create one, otherwise we will update existing one
216 if reviewBoardUrl:
217 # For review board REST APIs we need to get just the ID (the number)
218 linkSplit = reviewBoardUrl.split('/')
219 reviewId = linkSplit[len(linkSplit)-1]
220 print "Updating existing review request %s with new patch" % reviewId
221 # Review request itself
222 reviewRequest = rbRoot.get_review_request(review_request_id=reviewId)
223 # Update diff (the patch) and publish the changes
224 reviewRequest.get_diffs().upload_diff(get_file_content(patch))
225 draft = reviewRequest.get_draft()
226 draft.update(public=True)
227 else:
228 print "Creating new review request"
229 jiraSummary = jiraDetails.get('fields').get('summary')
230 jiraDescription = jiraDetails.get('fields').get('description')
231 # Create review request
232 reviewRequest = rbRoot.get_review_requests().create(repository=rbRepoId)
233 # Attach patch
234 reviewRequest.get_diffs().upload_diff(get_file_content(patch))
235 # And add details
236 draft = reviewRequest.get_draft()
237 draft = draft.update(
238 summary='%s: %s' % (options.jira, jiraSummary),
239 description=jiraDescription,
240 target_groups=options.rb_group,
241 target_people=options.rb_user,
242 bugs_closed=options.jira
243 )
244 draft.update(public=True)
245 linkSplit = draft.links.review_request.href.split('/')
246 reviewId = linkSplit[len(linkSplit)-2]
247 reviewBoardUrl = "%s/r/%s" % (options.rb_url, reviewId)
248 jira_post_links(reviewBoardUrl, options.jira_rb_label, options)
249 print "Created new review: %s" % reviewBoardUrl
250
251 # Verify state of the JIRA to see if it's in the right state
252 if jiraDetails.get("fields").get("status").get("name") != options.jira_transition:
253 # JIRA REST API needs transition ID and not the human readable name, so we have to translate it first
254 jiraTransitions = jira_get_transitions(options)
255 transitionId = -1
256 for transition in jiraTransitions.get("transitions"):
257 if transition.get("to").get("name") == options.jira_transition:
258 transitionId = transition.get("id")
259 if transitionId == -1:
260 exit("Did not find valid transition id for %s" % options.jira_transition)
261 else:
262 if options.verbose:
263 print "Transition id for transition %s is %s" % (options.jira_transition, transitionId)
264 # And finally switch to patch available state
265 jira_post_transitions(transitionId, options)
266 print "Switch JIRA %s to %s state" % (options.jira, options.jira_transition)
267 else:
268 if options.verbose:
269 print "JIRA %s is already in %s" % (options.jira, options.jira_transition)
270
271 # Upload generated patch to JIRA itself
272 jira_post_attachments(patch, options)
273
274 # And that's it!
275 print "And we're done!"