Skip to content

Commit 3645ed2

Browse files
mzimandlmzimandl
committed
conc download uses rq notification task to show backend error messages
1 parent cf0d408 commit 3645ed2

File tree

26 files changed

+107
-61
lines changed

26 files changed

+107
-61
lines changed

‎lib/action/control.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
from action.argmapping.action import create_mapped_args
2929
from action.errors import (
3030
AlignedCorpusForbiddenException, CorpusForbiddenException,
31-
ForbiddenException, ImmediateRedirectException, UserReadableException)
31+
ForbiddenException, ImmediateRedirectException, UnavailableForLegalReasons, UserReadableException)
3232
from action.krequest import KRequest
3333
from action.model import ModelsSharedData
3434
from action.model.abstract import AbstractPageModel, AbstractUserModel

‎lib/bgcalc/adapter/abstract.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ class AbstractBgClient(abc.ABC):
3939

4040
@abc.abstractmethod
4141
async def send_task(
42-
self, name, ans_type: Type[T], args=None, time_limit=None, soft_time_limit=None) -> AbstractResultWrapper:
42+
self, name, ans_type: Type[T], args=None, time_limit=None, soft_time_limit=None, task_id=None) -> AbstractResultWrapper:
4343
pass
4444

4545
@abc.abstractmethod

‎lib/bgcalc/adapter/rq.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -204,15 +204,15 @@ def _resolve_limit(softl, hardl):
204204
def control(self):
205205
return self._control
206206

207-
def send_task_sync(self, name, ans_type: Type[T], args=None, time_limit=None, soft_time_limit=None) -> ResultWrapper[T]:
207+
def send_task_sync(self, name, ans_type: Type[T], args=None, time_limit=None, soft_time_limit=None, task_id=None) -> ResultWrapper[T]:
208208
tl = self._resolve_limit(time_limit, soft_time_limit)
209209
try:
210-
job = self.queue.enqueue(f'{self.prefix}.{name}', job_timeout=tl, args=args)
210+
job = self.queue.enqueue(f'{self.prefix}.{name}', job_timeout=tl, args=args, job_id=task_id)
211211
return ResultWrapper(job)
212212
except Exception as ex:
213213
logging.getLogger(__name__).error(ex)
214214

215-
async def send_task(self, name, ans_type: Type[T], args=None, time_limit=None, soft_time_limit=None) -> ResultWrapper[T]:
215+
async def send_task(self, name, ans_type: Type[T], args=None, time_limit=None, soft_time_limit=None, task_id=None) -> ResultWrapper[T]:
216216
"""
217217
Send a task to the worker.
218218
@@ -221,7 +221,7 @@ async def send_task(self, name, ans_type: Type[T], args=None, time_limit=None, s
221221
selected. Otherwise, the non-None is applied.
222222
"""
223223
return await asyncio.get_event_loop().run_in_executor(
224-
None, self.send_task_sync, name, ans_type, args, time_limit, soft_time_limit)
224+
None, self.send_task_sync, name, ans_type, args, time_limit, soft_time_limit, task_id)
225225

226226
def get_task_error(self, task_id):
227227
try:

‎lib/bgcalc/freqs/storage.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
from conclib.freq import FreqData, FreqItem
2727
from dataclasses_json import dataclass_json
2828

29-
MAX_DATA_LEN_DIRECT_PROVIDING = 500
29+
MAX_DATA_LEN_DIRECT_PROVIDING = 10
3030

3131

3232
def _cache_dir_path(args: FreqCalcArgs) -> str:

‎lib/bgcalc/task.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ class AsyncTaskStatus:
3535
CATEGORY_FREQ_PRECALC: ClassVar[str] = 'freqPrecalc'
3636
CATEGORY_WORDLIST: ClassVar[str] = 'wordlist'
3737
CATEGORY_KWORDS: ClassVar[str] = 'kwords'
38+
CATEGORY_NOTIFICATION: ClassVar[str] = 'notification'
3839

3940
ident: str
4041
"task identifier (unique per specific task instance)"

‎lib/views/concordance.py‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,12 @@
2020
import re
2121
import time
2222
from dataclasses import asdict, dataclass
23-
from typing import Any, Callable, Dict, List, Optional, Union
23+
from typing import Any, Callable, Dict, List, Optional, Set, Union
2424

2525
import conclib
2626
import corplib
27+
from bgcalc.task import AsyncTaskStatus
28+
import bgcalc
2729
import mailing
2830
import plugins
2931
import settings
@@ -1237,6 +1239,7 @@ class SaveConcArgs:
12371239
align_kwic: int = 0
12381240
from_line: int = 0
12391241
to_line: IntOpt = -1
1242+
task_id: str = ''
12401243

12411244

12421245
def _get_ipm_base_set_desc(corp: AbstractKCorpus, contains_within, translate: Callable[[str], str]):
@@ -1259,7 +1262,7 @@ def _get_ipm_base_set_desc(corp: AbstractKCorpus, contains_within, translate: Ca
12591262
@bp.route('/saveconc')
12601263
@http_action(
12611264
access_level=2, action_model=ConcActionModel, mapped_args=SaveConcArgs, return_type='plain')
1262-
async def saveconc(amodel: ConcActionModel, req: KRequest[SaveConcArgs], resp: KResponse):
1265+
async def saveconc(amodel: ConcActionModel, req: KRequest[SaveConcArgs], resp: KResponse):
12631266
def mkfilename(suffix):
12641267
return f'{amodel.args.corpname}-concordance.{suffix}'
12651268

@@ -1292,6 +1295,7 @@ def mkfilename(suffix):
12921295
'Content-Disposition',
12931296
f'attachment; filename="{mkfilename(req.mapped_args.saveformat)}"')
12941297

1298+
long_lines: Set[int] = set()
12951299
for from_line, to_line in line_range_chunks:
12961300
if from_line > 0:
12971301
req.mapped_args.heading = 0
@@ -1316,12 +1320,18 @@ def mkfilename(suffix):
13161320

13171321
maxcontext = int(amodel.corp.get_conf('MAXCONTEXT'))
13181322
if maxcontext:
1319-
for line in data.Lines:
1323+
for i, line in enumerate(data.Lines, from_line+1):
13201324
if len(line["Kwic"]) > maxcontext:
1321-
raise UnavailableForLegalReasons('KWIC too large')
1325+
line["Kwic"] = line["Kwic"][:maxcontext] + [{'str': '...'}]
1326+
long_lines.add(i)
13221327
if len(data.Lines) > 0:
13231328
await writer.write_conc(amodel, data, req.mapped_args)
1329+
13241330
output = writer.raw_content()
1331+
1332+
worker = bgcalc.calc_backend_client(settings)
1333+
notification = f'KWIC exceeds max allowed length; Shortened lines: {", ".join(map(str, sorted(long_lines)))}' if long_lines else None
1334+
await worker.send_task('notification', object.__class__, (notification,), task_id = req.mapped_args.task_id)
13251335
return bytes_stream(output)
13261336

13271337
except Exception as e:

‎public/files/js/app/page.ts‎

Lines changed: 22 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ export function isDownloadType(s:string):s is DownloadType {
8888
}
8989

9090
export interface SaveLinkHandler<T = any> {
91-
(name:string, format:string, url:string, args?:T):void;
91+
(name:string, format:string, urlConstructor:(taskId:string) => string, args?:T):void;
9292
}
9393

9494
export class UnsupportedBlob implements Blob {
@@ -321,12 +321,12 @@ export abstract class PageModel implements Kontext.IURLHandler, IConcArgsHandler
321321
*
322322
*/
323323
bgDownload<T=Kontext.AjaxArgs>(
324-
{name, format, datasetType, url, contentType, args}:
324+
{name, format, datasetType, urlConstructor, contentType, args}:
325325
{
326326
name?:string,
327327
format:string,
328328
datasetType:DownloadType,
329-
url:string,
329+
urlConstructor:(taskId:string) => string,
330330
contentType:string,
331331
args?:T
332332
}):Observable<string> {
@@ -337,7 +337,8 @@ export abstract class PageModel implements Kontext.IURLHandler, IConcArgsHandler
337337
return `kontext-${name ? name + '-' : ''}${datasetType}-${dt}.${format}`;
338338
}
339339

340-
const taskId = `${new Date().getTime()}:${url}`;
340+
const taskId = `${new Date().getTime()}.${datasetType}`;
341+
const url = urlConstructor(taskId);
341342
const method = () => { // TODO this is an antipattern (should be part of download types)
342343
if (
343344
datasetType === DownloadType.FREQ2D ||
@@ -381,13 +382,23 @@ export abstract class PageModel implements Kontext.IURLHandler, IConcArgsHandler
381382
);
382383

383384
} else {
384-
this.dispatcher.dispatch(
385-
ATActions.InboxUpdateAsyncTask,
386-
{
387-
ident: taskId,
388-
status: 'SUCCESS'
389-
}
390-
);
385+
if (datasetType === DownloadType.CONCORDANCE) {
386+
this.dispatcher.dispatch(
387+
ATActions.AsyncTasksWatch,
388+
{
389+
ident: taskId,
390+
},
391+
);
392+
393+
} else {
394+
this.dispatcher.dispatch(
395+
ATActions.InboxUpdateAsyncTask,
396+
{
397+
ident: taskId,
398+
status: 'SUCCESS'
399+
}
400+
);
401+
}
391402
}
392403
}
393404
),

‎public/files/js/app/plugin.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -169,18 +169,18 @@ export class PluginApi implements IPluginApi {
169169
name,
170170
format,
171171
datasetType,
172-
url,
172+
urlConstructor,
173173
contentType,
174-
args}:
175-
{
174+
args,
175+
}: {
176176
name?:string,
177177
format:string,
178178
datasetType:DownloadType,
179-
url:string,
179+
urlConstructor:(taskId:string) => string,
180180
contentType:string,
181181
args?:T
182182
}
183183
):Observable<string> {
184-
return this.pageModel.bgDownload({name, format, datasetType, url, contentType, args});
184+
return this.pageModel.bgDownload({name, format, datasetType, urlConstructor, contentType, args});
185185
}
186186
}

‎public/files/js/models/asyncTask/actions.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,12 @@ export class Actions {
7272
name: 'ASYNC_TASKS_CHECKED'
7373
};
7474

75+
static AsyncTasksWatch:Action<{
76+
ident:string;
77+
}> = {
78+
name: 'ASYNC_TASKS_WATCH'
79+
};
80+
7581
static isAsyncTasksChecked(a:Action):a is typeof Actions.AsyncTasksChecked {
7682
return a.name === Actions.AsyncTasksChecked.name;
7783
}

‎public/files/js/models/asyncTask/index.ts‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -80,12 +80,12 @@ export class AsyncTaskChecker extends StatefulModel<AsyncTaskCheckerState> {
8080
Actions.AsyncTasksChecked.name,
8181
action => {
8282
this.changeState(state => {
83-
const updatedList:Array<Kontext.AsyncTaskInfo> = [
84-
...List.filter(v => isDownloadType(v.category), state.asyncTasks)];
83+
const updatedList:Array<Kontext.AsyncTaskInfo> = [...state.asyncTasks];
8584
List.forEach(
8685
newTask => {
87-
const updated = List.find(t => t.ident === newTask.ident, this.state.asyncTasks);
88-
updatedList.push(updated ? updated : newTask);
86+
if (List.find(t => t.ident === newTask.ident, updatedList) === undefined) {
87+
updatedList.push(newTask);
88+
}
8989
},
9090
action.payload.tasks
9191
);
@@ -154,7 +154,7 @@ export class AsyncTaskChecker extends StatefulModel<AsyncTaskCheckerState> {
154154
state.asyncTasks.push(newTask);
155155
});
156156
if (!Dict.hasValue(newTask.category, DownloadType)) {
157-
this.startWatchingTask(newTask);
157+
this.startWatchingTask(newTask.ident);
158158
}
159159

160160
} else {
@@ -163,6 +163,13 @@ export class AsyncTaskChecker extends StatefulModel<AsyncTaskCheckerState> {
163163
}
164164
);
165165

166+
this.addActionHandler(
167+
Actions.AsyncTasksWatch,
168+
action => {
169+
this.startWatchingTask(action.payload.ident);
170+
}
171+
);
172+
166173
this.addActionHandler(
167174
Actions.InboxUpdateAsyncTask,
168175
action => {
@@ -258,11 +265,11 @@ export class AsyncTaskChecker extends StatefulModel<AsyncTaskCheckerState> {
258265
return List.filter(taskIsFinished, state.asyncTasks);
259266
}
260267

261-
private startWatchingTask(task:Kontext.AsyncTaskInfo) {
268+
private startWatchingTask(ident:string) {
262269
this.pageModel.openEventSource<Kontext.AsyncTaskInfo>(
263270
this.pageModel.createActionUrl<{taskId: string}>(
264271
'task_status',
265-
{taskId: task.ident}
272+
{taskId: ident}
266273
),
267274
(v:Kontext.AsyncTaskInfo) => v.status === 'SUCCESS' || v.status === 'FAILURE'
268275

@@ -290,7 +297,7 @@ export class AsyncTaskChecker extends StatefulModel<AsyncTaskCheckerState> {
290297
// exclude download tasks
291298
List.filter(v => !Dict.hasValue(v.category, DownloadType)),
292299
List.forEach(item => {
293-
this.startWatchingTask(item);
300+
this.startWatchingTask(item.ident);
294301
})
295302
);
296303
}

0 commit comments

Comments
 (0)