Skip to content

Commit

Permalink
Merge branch 'master' into dependabot/npm_and_yarn/tough-cookie-4.1.3
Browse files Browse the repository at this point in the history
  • Loading branch information
@Harshg999
Harshg999 authored Jul 12, 2023
2 parents 052be95 + 40189a3 commit 9b334ca
Show file tree
Hide file tree
Showing 4 changed files with 102 additions and 49 deletions.
5 changes: 4 additions & 1 deletion apps/jobbrowser/src/jobbrowser/api2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,8 +94,11 @@ def job(request, interface=None):
app_id = json.loads(request.POST.get('app_id'))

if interface == 'schedules':
filters = dict([(key, value) for _filter in json.loads(
request.POST.get('filters', '[]')) for key, value in list(_filter.items()) if value
])
offset = json.loads(request.POST.get('pagination', '{"offset": 1}')).get('offset')
response_app = get_api(request.user, interface, cluster=cluster).app(app_id, offset=offset)
response_app = get_api(request.user, interface, cluster=cluster).app(app_id, offset=offset, filters=filters)
else:
response_app = get_api(request.user, interface, cluster=cluster).app(app_id)

Expand Down
68 changes: 45 additions & 23 deletions apps/jobbrowser/src/jobbrowser/apis/schedule_api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ def apps(self, filters):
jobs = oozie_api.get_coordinators(**kwargs)

return {
'apps':[{
'apps': [{
'id': app['id'],
'name': app['appName'],
'status': app['status'],
Expand All @@ -72,12 +72,34 @@ def apps(self, filters):
}


def app(self, appid, offset=1):
def app(self, appid, offset=1, filters={}):
oozie_api = get_oozie(self.user)
coordinator = oozie_api.get_coordinator(jobid=appid)

mock_get = MockGet()
mock_get.update('offset', offset)

"""
The Oozie job api supports one or more "status" parameters. The valid status values are:
WAITING, READY, SUBMITTED, RUNNING, SUSPENDED, TIMEDOUT, SUCCEEDED, KILLED, FAILED, IGNORED, SKIPPED
The job browser UI has a generic filter mechanism that is re-used across all different type of jobs, that
parameter is called "states" and it only has three possible values: completed, running or failed
Here we adapt this to fit the API requirements, "state" becomes "status" and the values are translated
based on how it's been done historically (for instance list_oozie_coordinator.mako around line 725).
"""
if 'states' in filters:
statusFilters = []
for stateFilter in filters.get('states'):
if stateFilter == 'completed':
statusFilters.append('SUCCEEDED')
elif stateFilter == 'running':
statusFilters.extend(['RUNNING', 'READY', 'SUBMITTED', 'SUSPENDED', 'WAITING'])
elif stateFilter == 'failed':
statusFilters.extend(['KILLED', 'FAILED', 'TIMEDOUT', 'SKIPPED'])
mock_get.update('status', statusFilters)
request = MockDjangoRequest(self.user, get=mock_get)
response = list_oozie_coordinator(request, job_id=appid)

Expand Down Expand Up @@ -133,36 +155,36 @@ def profile(self, appid, app_type, app_property, app_filters):
return coordinator['properties']['tasks']

_API_STATUSES = {
'PREP': 'RUNNING',
'RUNNING': 'RUNNING',
'RUNNINGWITHERROR': 'RUNNING',
'PREPSUSPENDED': 'PAUSED',
'SUSPENDED': 'PAUSED',
'PREP': 'RUNNING',
'RUNNING': 'RUNNING',
'RUNNINGWITHERROR': 'RUNNING',
'PREPSUSPENDED': 'PAUSED',
'SUSPENDED': 'PAUSED',
'SUSPENDEDWITHERROR': 'PAUSED',
'PREPPAUSED': 'PAUSED',
'PAUSED': 'PAUSED',
'PAUSEDWITHERROR': 'PAUSED',
'SUCCEEDED': 'SUCCEEDED',
'DONEWITHERROR': 'FAILED',
'KILLED': 'FAILED',
'FAILED': 'FAILED',
'PREPPAUSED': 'PAUSED',
'PAUSED': 'PAUSED',
'PAUSEDWITHERROR': 'PAUSED',
'SUCCEEDED': 'SUCCEEDED',
'DONEWITHERROR': 'FAILED',
'KILLED': 'FAILED',
'FAILED': 'FAILED',
}

def _api_status(self, status):
return self._API_STATUSES.get(status, 'FAILED')

_TASK_API_STATUSES = {
'WAITING': 'RUNNING',
'READY': 'RUNNING',
'WAITING': 'RUNNING',
'READY': 'RUNNING',
'SUBMITTED': 'RUNNING',
'RUNNING': 'RUNNING',
'RUNNING': 'RUNNING',
'SUSPENDED': 'PAUSED',
'SUCCEEDED': 'SUCCEEDED',
'TIMEDOUT': 'FAILED',
'KILLED': 'FAILED',
'FAILED': 'FAILED',
'IGNORED': 'FAILED',
'SKIPPED': 'FAILED',
'TIMEDOUT': 'FAILED',
'KILLED': 'FAILED',
'FAILED': 'FAILED',
'IGNORED': 'FAILED',
'SKIPPED': 'FAILED',
}

def _task_api_status(self, status):
Expand Down Expand Up @@ -192,4 +214,4 @@ def get(self, prop, default=None):
return self._prop.get(prop, default)

def getlist(self, prop):
return []
return self._prop.get(prop)
76 changes: 53 additions & 23 deletions apps/jobbrowser/src/jobbrowser/templates/job_browser.mako
View file
Original file line number Diff line number Diff line change
Expand Up @@ -492,7 +492,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
<!-- /ko -->
<!-- /ko -->
<!-- ko if: $root.job() && $root.job().hasPagination() && interface() === 'schedules' -->
<!-- ko if: $root.job() && !$root.job().forceUpdatingJob() && $root.job().hasPagination() && interface() === 'schedules' -->
<div data-bind="template: { name: 'pagination${ SUFFIX }', data: $root.job() }, visible: !jobs.loadingJobs()"></div>
<!-- /ko -->
<div data-bind="template: { name: 'pagination${ SUFFIX }', data: $root.jobs }, visible: !$root.job() && !jobs.loadingJobs()"></div>
Expand Down Expand Up @@ -2444,13 +2444,16 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
<!-- ko with: coordinatorActions() -->
<form class="form-inline">
##<input data-bind="value: textFilter" type="text" class="input-xlarge search-query" placeholder="${_('Filter by name')}">
##<span data-bind="foreach: statesValuesFilter">
## <label class="checkbox">
## <div class="pull-left margin-left-5 status-border status-content" data-bind="css: value, hueCheckbox: checked"></div>
## <div class="inline-block" data-bind="text: name, toggle: checked"></div>
## </label>
##</span>
<!-- ko with: $root.job() -->
<!-- ko if: type() === 'schedule' -->
<span data-bind="foreach: statesValuesFilter">
<label class="checkbox">
<div class="pull-left margin-left-5 status-border status-content" data-bind="css: value, hueCheckbox: checked"></div>
<div class="inline-block" data-bind="text: name, toggle: checked"></div>
</label>
</span>
<!-- /ko -->
<!-- /ko -->
<div data-bind="template: { name: 'job-actions${ SUFFIX }' }" class="pull-right"></div>
</form>
Expand All @@ -2470,6 +2473,7 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
<th>${_('lastModifiedTime')}</th>
</tr>
</thead>
<!-- ko if: !$root.job().forceUpdatingJob() -->
<tbody data-bind="foreach: apps">
<tr class="status-border pointer" data-bind="
css: {
Expand All @@ -2495,6 +2499,14 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
<td data-bind="text: properties.lastModifiedTime"></td>
</tr>
</tbody>
<!-- /ko -->
<!-- ko if: $root.job().forceUpdatingJob() -->
<tbody>
<tr>
<td colspan="11"><!-- ko hueSpinner: { spin: true, inline: true, size: 'large' } --><!-- /ko --></td>
</tr>
</tbody>
<!-- /ko -->
</table>
<!-- /ko -->
</div>
Expand Down Expand Up @@ -2806,9 +2818,9 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
self.paginationPage = ko.observable(1);
self.paginationOffset = ko.observable(1); // Starting index
self.paginationResultPage = ko.observable(50);
self.totalApps = ko.observable(null);
self.totalApps = ko.observable(job.properties && job.properties.total_actions || 0);
self.hasPagination = ko.computed(function() {
return ['workflows', 'schedules', 'bundles'].indexOf(vm.interface()) != -1;
return self.totalApps() && ['workflows', 'schedules', 'bundles'].indexOf(vm.interface()) !== -1;
});
self.pagination = ko.pureComputed(function() {
return {
Expand All @@ -2818,11 +2830,9 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
};
});
self.pagination.subscribe(function(value) {
if (vm.interface() === 'schedules' && value.page > 1) {
vm.interface('schedules');
self.hasPagination(true);
self.fetchJob();
self.pagination.subscribe(function() {
if (vm.interface() === 'schedules') {
self.updateJob(false, true);
}
});
Expand Down Expand Up @@ -2880,6 +2890,12 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
return ['RUNNING', 'PAUSED'].indexOf(self.apiStatus()) != -1 || job.isRunning;
});
self.isRunning.subscribe(function () {
// The JB page for jobs is split in two tables, "Running" and "Completed", this esentially unchecks any job
// that moves from one table to the other.
ko.utils.arrayRemoveItem(vm.jobs.selectedJobs(), self)
});
self.user = ko.observableDefault(job.user);
self.queue = ko.observableDefault(job.queue);
self.cluster = ko.observableDefault(job.cluster);
Expand Down Expand Up @@ -2963,8 +2979,13 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
{'types': ko.mapping.toJS(self.typesFilter())},
];
});
self.filters.subscribe(function(value) {
self.fetchProfile('tasks');
self.forceUpdatingJob = ko.observable(false);
self.filters.subscribe(function () {
if (self.type() === 'schedule') {
self.updateJob(false, true);
} else {
self.fetchProfile('tasks');
}
});
self.metadataFilter = ko.observable('');
self.metadataFilter.subscribe(function(newValue) {
Expand Down Expand Up @@ -3057,7 +3078,8 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
cluster: ko.mapping.toJSON(vm.compute),
app_id: ko.mapping.toJSON(self.id),
interface: ko.mapping.toJSON(vm.interface),
pagination: ko.mapping.toJSON(self.pagination)
pagination: ko.mapping.toJSON(self.pagination),
filters: ko.mapping.toJSON(self.filters)
}, function (data) {
if (data.status == 0) {
if (data.app) {
Expand Down Expand Up @@ -3250,11 +3272,14 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
});
};
self.updateJob = function (updateLogs) {
self.updateJob = function (updateLogs, forceUpdate) {
huePubSub.publish('graph.refresh.view');
var deferred = $.Deferred();
if (vm.job() == self && self.apiStatus() == 'RUNNING') {
if (vm.job() == self && (self.apiStatus() == 'RUNNING' || forceUpdate)) {
vm.apiHelper.cancelActiveRequest(lastUpdateJobRequest);
if (forceUpdate) {
self.forceUpdatingJob(true);
}
lastUpdateJobRequest = self._fetchJob(function (data) {
var requests = [];
if (['schedule', 'workflow'].indexOf(vm.job().type()) >= 0) {
Expand Down Expand Up @@ -3291,6 +3316,8 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
$.when.apply(this, requests).done(function (){
deferred.resolve();
});
}).always(function () {
self.forceUpdatingJob(false);
});
}
return deferred;
Expand Down Expand Up @@ -3644,9 +3671,12 @@ ${ commonheader("Job Browser", "jobbrowser", user, request) | n,unicode }
return self.isCoordinator();
});
self.rerunEnabled = ko.pureComputed(function() {
return self.hasRerun() && self.selectedJobs().length == 1 && $.grep(self.selectedJobs(), function(job) {
return job.rerunEnabled();
}).length == self.selectedJobs().length;
var validSelectionCount =
self.selectedJobs().length === 1 ||
(self.selectedJobs().length > 1 && vm.interface() === 'schedules');
return self.hasRerun() && validSelectionCount && $.grep(self.selectedJobs(), function (job) {
return job.rerunEnabled();
}).length === self.selectedJobs().length;
});
self.hasPause = ko.pureComputed(function() {
Expand Down
2 changes: 0 additions & 2 deletions desktop/core/base_requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,11 @@ kerberos==1.3.0
lockfile==0.12.2
Mako==1.2.3
Markdown==3.1
mysqlclient==2.1.1 # don't move ext-py3
nose==1.3.7
openpyxl==3.0.9
phoenixdb==1.2.1
prompt-toolkit==2.0.10
protobuf==3.20.3
psycopg2-binary==2.9.6
pyformance==0.3.2
pylint==2.6.0
pylint-django==2.3.0
Expand Down

0 comments on commit 9b334ca

Please sign in to comment.