Skip to content

Commit 29bbfee

Browse files
xarx00xarx00
committed
Added bulk ci-status
1 parent 3ee2e0c commit 29bbfee

File tree

1 file changed

+131
-52
lines changed

1 file changed

+131
-52
lines changed

‎gitlab/v4/bulk.py

+131-52
Original file line numberDiff line numberDiff line change
@@ -204,23 +204,29 @@ def local_projects(self, group_path=None, **kwargs):
204204
return [self.get_grpath(wdpath)
205205
for wdpath in self.wdprojects(group_path=group_path)]
206206

207-
@cli.register_custom_action('BulkManager', tuple(), ('group-path', ))
208-
def remote_projects(self, group_path=None, **kwargs):
209-
"""Returns the list of project paths of all remote projects
210-
under group-path.
207+
def _remote_projects(self, group_path=None, **kwargs):
208+
"""Returns the list of all remote projects under group-path
209+
as objects.
211210
"""
212211
print_progress("Collecting remote projects")
213212
try:
214213
grpath = group_path or self.workdir_group
215214
l = len(grpath)
216-
projects = [p.path_with_namespace
217-
for p in self.gitlab.projects.list(all=True, simple=True)
218-
if p.path_with_namespace == grpath or
219-
p.path_with_namespace.startswith(grpath) and
220-
p.path_with_namespace[l] == '/']
215+
remote = [p for p in self.gitlab.projects.list(all=True, simple=True)
216+
if p.path_with_namespace == grpath or
217+
p.path_with_namespace.startswith(grpath) and
218+
p.path_with_namespace[l] == '/']
221219
finally:
222220
print_progress()
223-
return projects
221+
return remote
222+
223+
@cli.register_custom_action('BulkManager', tuple(), ('group-path', ))
224+
def remote_projects(self, group_path=None, _remote=None, **kwargs):
225+
"""Returns the list of project paths of all remote projects
226+
under group-path.
227+
"""
228+
remote = _remote or self._remote_projects(group_path=group_path)
229+
return [p.path_with_namespace for p in remote]
224230

225231
def _get_projects(self, group_path=None):
226232
"""Returns the list of all local projects under group-path.
@@ -280,7 +286,7 @@ def get_submodule_remote_branch(self, sm, repo):
280286

281287
@cli.register_custom_action('BulkManager', tuple(),
282288
('group-path', 'no-remote', 'branch'))
283-
def errors(self, group_path=None, no_remote=False, branch=None,
289+
def git_errors(self, group_path=None, no_remote=False, branch=None,
284290
_projects=None, **kwargs):
285291
projects = _projects or self._get_projects(group_path=group_path)
286292
errors = {prpath:[] for (wdpath, prpath, repo) in projects}
@@ -342,13 +348,73 @@ def errors(self, group_path=None, no_remote=False, branch=None,
342348

343349

344350
@cli.register_custom_action('BulkManager', tuple(),
345-
('group-path', 'no-remote', 'branch'))
346-
def status(self, group_path=None, no_remote=False, branch=None,
347-
**kwargs):
351+
('group-path', 'branch'))
352+
def ci_status(self, group_path=None, branch=None,
353+
_projects=None, _remote=None, **kwargs):
354+
projects = _projects or self._get_projects(group_path=group_path)
355+
local = [prpath for (wdpath, prpath, repo) in projects]
356+
remote = _remote or self._remote_projects(group_path=group_path)
357+
remote = [p for p in remote if p.path_with_namespace in local]
358+
359+
statuses = {}
360+
def add_status(status, prpath):
361+
projects = statuses.get(status)
362+
if projects is None:
363+
projects = []
364+
statuses[status] = projects
365+
projects.append(prpath)
366+
367+
for p in remote:
368+
prpath = p.path_with_namespace
369+
print_progress("Processing CI status: " + prpath)
370+
371+
#find last relevant pipeline
372+
bpipelines = p.pipelines.list(scope="branches", ref=branch,
373+
#take only the last run
374+
page=1, per_page=1, sort="desc")
375+
tpipelines = p.pipelines.list(scope="tags",
376+
#take only the last run
377+
page=1, per_page=1, sort="desc")
378+
pipelines = bpipelines + tpipelines
379+
if not pipelines:
380+
add_status("no-pipeline", prpath)
381+
continue
382+
pipelines.sort(key=lambda x: x.id, reverse=True)
383+
pipeline = pipelines[0]
384+
385+
#check the pipeline status
386+
if pipeline.status != "failed":
387+
add_status(pipeline.status, prpath)
388+
continue
389+
#failed
390+
pipeline = p.pipelines.get(id=pipeline.id)
391+
if pipeline.yaml_errors:
392+
add_status("yaml-errors", prpath)
393+
continue
394+
jobs = pipeline.jobs.list(scope="failed",
395+
#take only the last failed job
396+
page=1, per_page=1, sort="desc")
397+
job = jobs[0]
398+
add_status("failed in stage '%s'" % job.stage, prpath)
399+
400+
print_progress()
401+
return statuses
402+
403+
404+
@cli.register_custom_action('BulkManager', tuple(),
405+
('group-path', 'branch', 'no-remote',
406+
'no-push-status', 'no-pull-status',
407+
'no-git-errors', 'no-ci-errors'))
408+
def status(self, group_path=None, branch=None, no_remote=False,
409+
no_push_status=False, no_pull_status=False,
410+
no_git_errors=False, no_ci_errors=False, **kwargs):
348411
projects = self._get_projects(group_path=group_path)
349412

350-
errors = self.errors(group_path=group_path, no_remote=no_remote,
351-
branch=branch, _projects=projects).keys()
413+
if no_git_errors:
414+
git_errors = None
415+
else:
416+
git_errors = self.git_errors(group_path=group_path, no_remote=no_remote,
417+
branch=branch, _projects=projects).keys()
352418
modified = []
353419
untracked = []
354420
for (wdpath, prpath, repo) in projects:
@@ -360,60 +426,72 @@ def status(self, group_path=None, no_remote=False, branch=None,
360426

361427
#Non-pushed repos
362428
non_pushed = []
363-
for (wdpath, prpath, repo) in projects:
364-
print_progress("Processing push status: " + prpath)
365-
needs_push = False
366-
response = repo.git.branch(v=True)
367-
if not repo.head.is_detached:
368-
pat_start = r'^(?:\*|\s*' + repo.active_branch.name + r'\s)'
369-
else:
370-
pat_start = r'^\*'
371-
pat = re.compile(pat_start + r'[^\[]*\[ahead', re.M)
372-
if pat.search(response):
373-
needs_push = True
374-
else:
375-
for sm in repo.submodules:
376-
if sm.module_exists():
377-
smrepo = sm.module()
378-
response = smrepo.git.branch(v=True)
379-
sm_branch = self.get_submodule_remote_branch(sm, repo)
380-
if sm_branch is not None:
381-
pat = re.compile(r'^\*?\s*' + sm_branch + r'[^\[]*\[ahead', re.M)
382-
if pat.search(response):
383-
needs_push = True
384-
break
385-
if needs_push:
386-
non_pushed.append(prpath)
387-
repo.git.clear_cache()
429+
if not no_push_status:
430+
for (wdpath, prpath, repo) in projects:
431+
print_progress("Processing push status: " + prpath)
432+
needs_push = False
433+
response = repo.git.branch(v=True)
434+
if not repo.head.is_detached:
435+
pat_start = r'^(?:\*|\s*' + repo.active_branch.name + r'\s)'
436+
else:
437+
pat_start = r'^\*'
438+
pat = re.compile(pat_start + r'[^\[]*\[ahead', re.M)
439+
if pat.search(response):
440+
needs_push = True
441+
else:
442+
for sm in repo.submodules:
443+
if sm.module_exists():
444+
smrepo = sm.module()
445+
response = smrepo.git.branch(v=True)
446+
sm_branch = self.get_submodule_remote_branch(sm, repo)
447+
if sm_branch is not None:
448+
pat = re.compile(r'^\*?\s*' + sm_branch + r'[^\[]*\[ahead', re.M)
449+
if pat.search(response):
450+
needs_push = True
451+
break
452+
if needs_push:
453+
non_pushed.append(prpath)
454+
repo.git.clear_cache()
388455

389456
if no_remote:
390-
local_only = remote_only = outdated = None
457+
local_only = remote_only = outdated = ci_errors = None
391458
else:
392459
print_progress("Processing remote data")
393460
local = [prpath for (wdpath, prpath, repo) in projects]
394-
remote = self.remote_projects(group_path=group_path)
461+
_remote = self._remote_projects(group_path=group_path)
462+
remote = self.remote_projects(group_path=group_path, _remote=_remote)
395463
#TODO local_only: compare repo remote location, not prpath
396464
local_only = [prpath for prpath in local if prpath not in remote]
397465
remote_only = [prpath for prpath in remote if prpath not in local]
398466

399467
outdated = []
400-
remote_name = self.gitlab.remote_name
401-
for (wdpath, prpath, repo) in projects:
402-
print_progress("Processing remote status: " + prpath)
403-
if prpath in remote:
404-
response = repo.git.remote('show', remote_name)
405-
if response.find('out of date') != -1:
406-
outdated.append(prpath)
468+
if not no_pull_status:
469+
remote_name = self.gitlab.remote_name
470+
for (wdpath, prpath, repo) in projects:
471+
print_progress("Processing remote status: " + prpath)
472+
if prpath in remote:
473+
response = repo.git.remote('show', remote_name)
474+
if response.find('out of date') != -1:
475+
outdated.append(prpath)
476+
if not no_ci_errors:
477+
ci_status = self.ci_status(group_path=group_path, branch=branch,
478+
_projects=projects, _remote=_remote)
479+
ci_errors = []
480+
for key, errs in ci_status.items():
481+
if key.startswith('failed') or key == 'yaml-errors':
482+
ci_errors += errs
483+
ci_errors = list(sorted(set(ci_errors)))
407484

408485
print_progress()
409486
status = {}
410-
if errors: status["errors"] = errors
487+
if git_errors: status["git-errors"] = git_errors
411488
if modified: status["modified"] = modified
412-
if untracked: status["untracked_files"] = untracked
489+
if untracked: status["untracked-files"] = untracked
413490
if non_pushed: status["need_push"] = non_pushed
414491
if outdated: status["outdated"] = outdated
415492
if local_only: status["local-only"] = local_only
416493
if remote_only: status["remote-only"] = remote_only
494+
if ci_errors: status["ci-errors"] = ci_errors
417495
return status
418496

419497

@@ -425,6 +503,7 @@ def clone(self, group_path=None, **kwargs):
425503

426504
for (wdpath, prpath) in projects:
427505
try:
506+
#TODO: "project exists" is not an erro, instead return the list of updated projects
428507
if os.path.exists(os.path.join(wdpath, '.git')):
429508
errors[prpath].append('Project already exists.')
430509
else:

0 commit comments

Comments
 (0)