Qlik Community
Collaborate with over 60,000 Qlik technologists and members around the world to get answers to your questions, and maximize success.
Join Us
// Add Task Trigger & Dependency details
LIB CONNECT TO 'monitor_apps_REST_event';
RestConnectorMasterTable:
SQL SELECT
"createdDate" AS "createdDate_u1",
"modifiedDate" AS "modifiedDate_u1",
"modifiedByUserName" AS "modifiedByUserName_u1",
"name" AS "name_u2",
"enabled" AS "enabled_u2",
"eventType",
"startDate",
"expirationDate",
"incrementDescription",
"incrementOption",
"__KEY_root",
(SELECT
"id" AS "id_u5", // For compound triggers
"__KEY_compositeRules",
"__FK_compositeRules",
(SELECT
"id" AS "id_u3",
"__FK_reloadTask",
"__KEY_reloadTask"
FROM "reloadTask" PK "__KEY_reloadTask" FK "__FK_reloadTask")
FROM "compositeRules" PK "__KEY_compositeRules" FK "__FK_compositeRules"),
(SELECT
"id" AS "id_u10",
"__FK_reloadTask_u0",
"__KEY_reloadTask_u0"
FROM "reloadTask" PK "__KEY_reloadTask_u0" FK "__FK_reloadTask_u0"),
(SELECT
"id" AS "id_u14",
"__FK_userSyncTask",
"__KEY_userSyncTask"
FROM "userSyncTask" PK "__KEY_userSyncTask" FK "__FK_userSyncTask")
FROM JSON (wrap on) "root" PK "__KEY_root";
// To get Task Name (of task which has an UPSTREAM dependency in a trigger), link __FK_compositeRule with __KEY_reloadTask_u0 to return id_u10 as TaskId
map_reloadTask:
Mapping LOAD
__KEY_reloadTask_u0 AS _reloadTask,
id_u10 AS taskIdReloadTask
RESIDENT RestConnectorMasterTable
WHERE NOT IsNull([__KEY_reloadTask_u0]);
// To get Task Name of the actual upstream dependency (aka "preceding task"), link __KEY_compositeRule with __FK_reloadTask to return id_u3 as TaskId
map_precedingTask:
Mapping LOAD
__FK_reloadTask AS _precedingTask,
id_u3 AS taskIdPrecedingTask
RESIDENT RestConnectorMasterTable
WHERE NOT IsNull([__FK_reloadTask]);
// Create composite task dependency for Task Triggers involving more than one task (e.g. Task2 reloads after Task0 and Task1 - which are independent from each other)
composite1:
LOAD
__KEY_root,
taskIdPrecedingTask,
ApplyMap('map_taskName2',taskIdPrecedingTask,null()) as compRulePrecedingTaskName
;
LOAD
[__FK_compositeRules] AS [__KEY_root],
ApplyMap('map_precedingTask',__KEY_compositeRules,null()) AS taskIdPrecedingTask
RESIDENT RestConnectorMasterTable
WHERE NOT IsNull([__FK_compositeRules]);
taskId1:
LOAD [__FK_reloadTask_u0] AS [__KEY_root],
[id_u10] AS taskId
RESIDENT RestConnectorMasterTable
WHERE NOT IsNull([__FK_reloadTask_u0]);
Concatenate (taskId1)
LOAD
[__FK_userSyncTask] AS [__KEY_root],
[id_u14] AS taskId
RESIDENT RestConnectorMasterTable
WHERE NOT IsNull([__FK_userSyncTask]);
map_taskId:
mapping Load
__KEY_root,
taskId
resident taskId1;
Drop Table taskId1;
TaskTrigger:
LOAD
*,
TaskId as tempId, // For tasks w/ no trigger, as explained below
ApplyMap('map_taskName2',TaskId,null()) as [TaskTriggerTaskName]
;
LOAD
__KEY_root,
date(alt(
date#(left(modifiedDate_u1,10),'YYYY-MM-DD'),
date#(left(modifiedDate_u1,10),'YYYY/MM/DD'),
date#(left(modifiedDate_u1,10),'MM-DD-YYYY'),
date#(left(modifiedDate_u1,10),'MM/DD/YYYY'),
date#(left(modifiedDate_u1,10),'YYYY.MM.DD'),
'No valid date')
) AS [Task Trigger Modified],
[modifiedByUserName_u1] AS [Task Trigger Modified By],
IF(left(expirationDate,4)=9999,'No expiration',
date(alt(
date#(left(expirationDate,10),'YYYY-MM-DD'),
date#(left(expirationDate,10),'YYYY/MM/DD'),
date#(left(expirationDate,10),'MM-DD-YYYY'),
date#(left(expirationDate,10),'MM/DD/YYYY'),
date#(left(expirationDate,10),'YYYY.MM.DD'),
'No expiration')
)) AS [Task Trigger Expiration Date],
[incrementDescription] AS [Task Trigger Increment Description],
[incrementOption] AS [Task Trigger Increment Option],
[name_u2] AS [Task Trigger Name],
[eventType] AS [Task Trigger Type],
[enabled_u2] AS [Task Trigger Enabled],
ApplyMap('map_taskId',__KEY_root,null()) as TaskId
RESIDENT RestConnectorMasterTable
WHERE NOT IsNull([__KEY_root]);
DROP TABLE RestConnectorMasterTable;
JOIN (TaskTrigger)
LOAD
__KEY_root,
compRulePrecedingTaskName as TaskTriggerDependencyTaskName ,
taskIdPrecedingTask as TaskTriggerTaskId
RESIDENT composite1;
DROP TABLE composite1;
// Concatenate these tasks for those tasks which have no triggers (like Manual Reload of ...)
// Without this logic, any task which has downstream dependencies but which has no trigger will not appear in the Task Dependencies page
Concatenate (TaskTrigger)
Load
// 0 as TaskTriggerCompositeCount,
TaskTriggerTaskId as TaskId,
TaskTriggerDependencyTaskName as TaskTriggerTaskName
RESIDENT TaskTrigger
WHERE not isnull (TaskTriggerTaskId)
AND Not Exists (tempId,TaskTriggerTaskId);
// A last ditch effort to make sure we show all Tasks in the Task Dependency charts
IF $(countOfReloads) = 0 then
Concatenate (TaskTrigger)
Load
0 as TaskTriggerCompositeCount,
TaskId,
[Task Name] as TaskTriggerTaskName
RESIDENT Task
WHERE Not Exists (tempId,TaskId);
ELSE
Concatenate (TaskTrigger)
Load
0 as TaskTriggerCompositeCount,
TaskId,
[Task Name] as TaskTriggerTaskName
RESIDENT TaskName
WHERE Not Exists (tempId,TaskId);
ENDIF
DROP FIELD tempId;
// Build Task Hierarchy
Staging:
Load
DISTINCT TaskId,
TaskTriggerTaskName AS TaskName,
TaskTriggerTaskId AS ParentTaskId,
TaskTriggerDependencyTaskName AS ParentTaskName
Resident TaskTrigger;
TaskHierarchy:
Hierarchy (TaskId, ParentTaskId, "Task Hierarchy", ParentTaskName, "Task Hierarchy", TaskPath, ' >> ', TaskDepth)
Load
TaskId,
ParentTaskId,
TaskName as [Task Hierarchy]
RESIDENT Staging;
TaskTree:
HierarchyBelongsTo (TaskId, ParentTaskId, "Task Downstream", TaskTreeID, TaskTreeName)
Load
TaskId,
ParentTaskId,
TaskName as [Task Downstream]
RESIDENT Staging;
drop table Staging;
maxTaskDepth:
Load max(TaskDepth) as maxTaskDepth Resident TaskHierarchy;
LET maxTaskDepth = peek('maxTaskDepth');
// Calculate Median, Avg, Max reload time for task chains
// 1- Reduce TaskTree to distinct values (no duplicates)
TaskTreeDistinct:
NoConcatenate Load
TaskId,
TaskTreeID,
MaxString(TaskTreeName) as TaskTreeName,
MaxString([Task Downstream]) as [Task Downstream]
RESIDENT TaskTree
Group By TaskId, TaskTreeID;
Drop Table TaskTree;
Rename Table TaskTreeDistinct to TaskTree;
// 2- Get reload stats for tasks in task chains for last 28 days (to keep this 'group by' manageable
IF $(countOfReloads)>0 THEN
Left Join (TaskTree) // We want this data on the TaskTree table for easy TaskTree analysis
Load
_reloadSummaryTaskId as TaskId, // Join on this field
Round(Median([Reload Duration]),0.02) as TaskTreeMedianDuration,
Round(Avg([Reload Duration]),0.02) as TaskTreeAverageDuration,
Round(Max([Reload Duration]),0.02) as TaskTreeMaxDuration
RESIDENT ReloadSummary
WHERE [Reload Finish] >= today(1)-28
Group By _reloadSummaryTaskId;
Drop Field _reloadSummaryTaskId; // Field no longer needed on Reload Summary table
Collaborate with over 60,000 Qlik technologists and members around the world to get answers to your questions, and maximize success.
Join UsSearch Qlik's Support Knowledge database or request assisted support for highly complex issues.
Submit a caseExperiencing a serious issue, please contact us by phone. For Data Integration related issues please refer to your onboarding documentation for current phone number.
Call Us