helix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j...@apache.org
Subject [01/14] helix git commit: Workflow: job dag for generic workflows
Date Sat, 18 Aug 2018 00:20:01 GMT
Repository: helix
Updated Branches:
  refs/heads/master 48e56fa76 -> f1c503712


Workflow: job dag for generic workflows


Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/b486d784
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/b486d784
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/b486d784

Branch: refs/heads/master
Commit: b486d7849a017fe4a5c1df6fcc7ebb18a6c646d1
Parents: 48e56fa
Author: Vivo Xu <vxu@linkedin.com>
Authored: Thu Feb 1 18:13:46 2018 -0800
Committer: Vivo Xu <vxu@linkedin.com>
Committed: Wed Aug 8 15:06:37 2018 -0700

----------------------------------------------------------------------
 .../app/workflow/shared/workflow.model.ts       |  18 +--
 .../workflow-dag/workflow-dag.component.html    |  53 +++++++++
 .../workflow-dag/workflow-dag.component.scss    |  13 +++
 .../workflow-dag/workflow-dag.component.spec.ts |  33 ++++++
 .../workflow-dag/workflow-dag.component.ts      |  63 ++++++++++
 .../workflow-detail.component.html              | 115 +++++++++----------
 .../workflow-detail.component.ts                |   7 +-
 .../client/app/workflow/workflow.module.ts      |  10 +-
 helix-front/package.json                        |   3 +
 9 files changed, 245 insertions(+), 70 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/helix/blob/b486d784/helix-front/client/app/workflow/shared/workflow.model.ts
----------------------------------------------------------------------
diff --git a/helix-front/client/app/workflow/shared/workflow.model.ts b/helix-front/client/app/workflow/shared/workflow.model.ts
index 096af32..363c8a7 100644
--- a/helix-front/client/app/workflow/shared/workflow.model.ts
+++ b/helix-front/client/app/workflow/shared/workflow.model.ts
@@ -9,18 +9,22 @@ export class Job {
   readonly rawName: string;
   readonly startTime: string;
   readonly state: string;
+  readonly parents: string[];
 
   constructor(
     rawName: string,
     workflowName: string,
     startTime: string,
-    state: string
+    state: string,
+    parents: string[]
   ) {
     this.rawName = rawName;
     // try to reduce the name
     this.name = _.replace(rawName, workflowName + '_', '');
     this.startTime = startTime;
     this.state = state;
+    // try to reduce parent names
+    this.parents = _.map(parents, parent => _.replace(parent, workflowName + '_', ''));
   }
 }
 
@@ -28,9 +32,8 @@ export class Workflow {
   readonly name: string;
   readonly config: any;
   readonly jobs: Job[];
-  // TODO vxu: will use a better structure for this
-  readonly parentJobs: any[];
   readonly context: any;
+  readonly json: any;
 
   get isJobQueue(): boolean {
     return this.config && this.config.IsJobQueue.toLowerCase() == 'true';
@@ -41,14 +44,14 @@ export class Workflow {
   }
 
   constructor(obj: any) {
+    this.json = obj;
     this.name = obj.id;
     this.config = obj.WorkflowConfig;
     this.context = obj.WorkflowContext;
-    this.jobs = this.parseJobs(obj.Jobs);
-    this.parentJobs = obj.ParentJobs;
+    this.jobs = this.parseJobs(obj.Jobs, obj.ParentJobs);
   }
 
-  protected parseJobs(list: string[]): Job[] {
+  protected parseJobs(list: string[], parents: any): Job[] {
     let result: Job[] = [];
 
     _.forEach(list, jobName => {
@@ -56,7 +59,8 @@ export class Workflow {
         jobName,
         this.name,
         _.get(this.context, ['StartTime', jobName]),
-        _.get(this.context, ['JOB_STATES', jobName])
+        _.get(this.context, ['JOB_STATES', jobName]),
+        parents[jobName]
       ));
     });
 

http://git-wip-us.apache.org/repos/asf/helix/blob/b486d784/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.html
----------------------------------------------------------------------
diff --git a/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.html b/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.html
new file mode 100644
index 0000000..c44d369
--- /dev/null
+++ b/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.html
@@ -0,0 +1,53 @@
+<ngx-graph
+  #graph
+  class="chart-container"
+  [view]="[graph.graphDims.width * 2 / 3, graph.graphDims.height]"
+  [links]="data.links"
+  [nodes]="data.nodes"
+  [curve]="curve"
+  orientation="TB"
+  (select)="select($event)"
+  [autoZoom]="false"
+  [panningEnabled]="false"
+  [draggingEnabled]="false"
+  [minZoomLevel]="1"
+  [maxZoomLevel]="1">
+
+  <ng-template #defsTemplate>
+    <svg:marker id="arrow" viewBox="0 -5 10 10" refX="8" refY="0" markerWidth="4" markerHeight="4"
orient="auto">
+      <svg:path d="M0,-5L10,0L0,5" class="arrow-head" />
+    </svg:marker>
+  </ng-template>
+
+  <ng-template #nodeTemplate let-node>
+    <svg:g class="node"
+      ngx-tooltip
+      [tooltipPlacement]="'top'"
+      [tooltipType]="'tooltip'"
+      [tooltipTitle]="node.description">
+      <svg:rect [attr.width]="node.width" [attr.height]="node.height" [attr.fill]="node.options.color"
/>
+      <svg:text alignment-baseline="central" [attr.x]="10" [attr.y]="node.height / 2">{{
node.label }}</svg:text>
+      <svg:text alignment-baseline="hanging" [attr.x]="node.width - 100" [attr.y]="node.height
+ 10" [ngClass]="'state-default state-' + node.state">{{ node.state }}</svg:text>
+    </svg:g>
+  </ng-template>
+
+  <ng-template #linkTemplate let-link>
+    <svg:g class="edge">
+      <svg:path
+        class="line"
+        stroke-width="2"
+        marker-end="url(#arrow)" >
+      </svg:path>
+      <svg:text class="edge-label" text-anchor="middle">
+        <textPath
+          class="text-path"
+          [attr.href]="'#' + link.id"
+          [style.dominant-baseline]="link.dominantBaseline"
+          startOffset="50%">
+          {{link.label}}
+        </textPath>
+      </svg:text>
+    </svg:g>
+  </ng-template>
+
+</ngx-graph>

http://git-wip-us.apache.org/repos/asf/helix/blob/b486d784/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.scss
----------------------------------------------------------------------
diff --git a/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.scss b/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.scss
new file mode 100644
index 0000000..87fd120
--- /dev/null
+++ b/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.scss
@@ -0,0 +1,13 @@
+@import '~@angular/material/theming';
+
+.state-default {
+  fill: mat-color(mat-palette($mat-deep-orange));
+}
+
+.state-COMPLETED {
+  fill: mat-color(mat-palette($mat-green));
+}
+
+.state-PENDING {
+  fill: mat-color(mat-palette($mat-grey));
+}

http://git-wip-us.apache.org/repos/asf/helix/blob/b486d784/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.spec.ts
----------------------------------------------------------------------
diff --git a/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.spec.ts b/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.spec.ts
new file mode 100644
index 0000000..40b600c
--- /dev/null
+++ b/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.spec.ts
@@ -0,0 +1,33 @@
+import { async, ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { NO_ERRORS_SCHEMA } from '@angular/core';
+
+import { TestingModule } from '../../../testing/testing.module';
+import { WorkflowDagComponent } from './workflow-dag.component';
+
+describe('WorkflowDagComponent', () => {
+  let component: WorkflowDagComponent;
+  let fixture: ComponentFixture<WorkflowDagComponent>;
+
+  beforeEach(async(() => {
+    TestBed.configureTestingModule({
+      imports: [ TestingModule ],
+      declarations: [ WorkflowDagComponent ],
+      schemas: [
+        /* avoid importing modules */
+        NO_ERRORS_SCHEMA
+      ]
+    })
+    .compileComponents();
+  }));
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(WorkflowDagComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  xit('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});

http://git-wip-us.apache.org/repos/asf/helix/blob/b486d784/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.ts
----------------------------------------------------------------------
diff --git a/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.ts b/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.ts
new file mode 100644
index 0000000..e94d913
--- /dev/null
+++ b/helix-front/client/app/workflow/workflow-dag/workflow-dag.component.ts
@@ -0,0 +1,63 @@
+import { Component, OnInit, Input, ElementRef, AfterViewInit, ViewChild } from '@angular/core';
+
+import * as shape from 'd3-shape';
+import * as _ from 'lodash';
+
+import { Workflow, Job } from '../shared/workflow.model';
+
+@Component({
+  selector: 'hi-workflow-dag',
+  templateUrl: './workflow-dag.component.html',
+  styleUrls: ['./workflow-dag.component.scss']
+})
+export class WorkflowDagComponent implements OnInit, AfterViewInit {
+
+  @Input()
+  workflow: Workflow;
+  curve: any = shape.curveLinear;
+  view = [800, 600];
+  data = {
+    nodes: [],
+    links: []
+  };
+  jobNameToId = {};
+
+  @ViewChild('graph')
+  graph;
+
+  constructor(protected el:ElementRef) { }
+
+  ngOnInit() {
+    this.loadJobs();
+  }
+
+  ngAfterViewInit() {
+    // console.log(this.el);
+    // console.log(this.graph);
+  }
+
+  protected loadJobs() {
+    // process nodes
+    _.forEach(this.workflow.jobs, (job: Job) => {
+      const newId = (this.data.nodes.length + 1).toString();
+
+      this.data.nodes.push({
+        id: newId,
+        label: job.name,
+        description: job.rawName,
+        state: job.state
+      });
+      this.jobNameToId[job.name] = newId;
+    });
+
+    // process edges/links
+    _.forEach(this.workflow.jobs, (job: Job) => {
+      _.forEach(job.parents, parentName => {
+        this.data.links.push({
+          source: this.jobNameToId[parentName],
+          target: this.jobNameToId[job.name]
+        });
+      });
+    });
+  }
+}

http://git-wip-us.apache.org/repos/asf/helix/blob/b486d784/helix-front/client/app/workflow/workflow-detail/workflow-detail.component.html
----------------------------------------------------------------------
diff --git a/helix-front/client/app/workflow/workflow-detail/workflow-detail.component.html
b/helix-front/client/app/workflow/workflow-detail/workflow-detail.component.html
index 0791995..5a940d3 100644
--- a/helix-front/client/app/workflow/workflow-detail/workflow-detail.component.html
+++ b/helix-front/client/app/workflow/workflow-detail/workflow-detail.component.html
@@ -18,69 +18,68 @@
 <section fxLayout="column" fxLayoutAlign="center center">
   <mat-spinner *ngIf="isLoading"></mat-spinner>
   <section *ngIf="!isLoading" class="content" fxLayout="column" fxLayoutAlign="center
center" fxLayoutGap="10px" fxFlexFill>
-    <mat-button-toggle-group #group="matButtonToggleGroup" value="queue">
-      <mat-button-toggle value="queue">
-        Queue View
+    <mat-button-toggle-group #group="matButtonToggleGroup" [value]="workflow.isJobQueue
? 'list' : 'graph'">
+      <mat-button-toggle *ngIf="!workflow.isJobQueue" value="graph">
+        Graph View
+      </mat-button-toggle>
+      <mat-button-toggle value="list">
+        List View
       </mat-button-toggle>
       <mat-button-toggle value="json">
         JSON View
       </mat-button-toggle>
     </mat-button-toggle-group>
-    <section class="viewer" [ngSwitch]="group.value" fxFlexFill>
-      <section *ngSwitchCase="'queue'">
-        <section *ngIf="workflow.isJobQueue">
-          <ngx-datatable
-            #jobsTable
-            class="material"
-            [headerHeight]="headerHeight"
-            [rowHeight]="rowHeight"
-            columnMode="force"
-            [footerHeight]="rowHeight"
-            [rows]="workflow.jobs"
-            selectionType="single"
-            [sorts]="sorts"
-            (select)="onSelect($event)"
-            [messages]="messages">
-            <ngx-datatable-column
-              name="Start Time"
-              [width]="200"
-              [resizeable]="false"
-              [draggable]="false"
-              [canAutoResize]="false">
-              <ng-template let-value="value" ngx-datatable-cell-template>
-                <span *ngIf="value" [matTooltip]="value | date:'medium'">
-                  {{ parseTime(value) }}
-                </span>
-                <span *ngIf="!value">-</span>
-              </ng-template>
-            </ngx-datatable-column>
-            <ngx-datatable-column name="Job Name" prop="name">
-              <ng-template let-row="row" let-value="value" ngx-datatable-cell-template>
-                <span [matTooltip]="row.rawName">
-                  ...{{ value }}
-                </span>
-              </ng-template>
-            </ngx-datatable-column>
-            <ngx-datatable-column
-              name="State"
-              [width]="120"
-              [resizeable]="false"
-              [draggable]="false"
-              [canAutoResize]="false">
-              <ng-template let-value="value" ngx-datatable-cell-template>
-                <span *ngIf="value" class="state-default state-{{ value }}">
-                  {{ value }}
-                </span>
-                <span *ngIf="!value" class="state-PENDING">PENDING</span>
-              </ng-template>
-            </ngx-datatable-column>
-          </ngx-datatable>
-        </section>
-        <section *ngIf="!workflow.isJobQueue">
-          {{ workflow | json }}
-        </section>
-      </section>
-      <ngx-json-viewer *ngSwitchCase="'json'" [json]="workflow"></ngx-json-viewer>
+    <section class="viewer" [ngSwitch]="group.value" fxLayout="column" fxLayoutAlign="center
center" fxFill>
+      <hi-workflow-dag *ngSwitchCase="'graph'" [workflow]="workflow"></hi-workflow-dag>
+      <ngx-datatable
+        *ngSwitchCase="'list'"
+        #jobsTable
+        class="material"
+        [headerHeight]="headerHeight"
+        [rowHeight]="rowHeight"
+        columnMode="force"
+        [footerHeight]="rowHeight"
+        [rows]="workflow.jobs"
+        selectionType="single"
+        [sorts]="sorts"
+        (select)="onSelect($event)"
+        [messages]="messages"
+        fxFill>
+        <ngx-datatable-column
+          name="Start Time"
+          [width]="200"
+          [resizeable]="false"
+          [draggable]="false"
+          [canAutoResize]="false">
+          <ng-template let-value="value" ngx-datatable-cell-template>
+            <span *ngIf="value" [matTooltip]="value | date:'medium'">
+              {{ parseTime(value) }}
+            </span>
+            <span *ngIf="!value">-</span>
+          </ng-template>
+        </ngx-datatable-column>
+        <ngx-datatable-column name="Job Name" prop="name">
+          <ng-template let-row="row" let-value="value" ngx-datatable-cell-template>
+            <span [matTooltip]="row.rawName">
+              ...{{ value }}
+            </span>
+          </ng-template>
+        </ngx-datatable-column>
+        <ngx-datatable-column
+          name="State"
+          [width]="120"
+          [resizeable]="false"
+          [draggable]="false"
+          [canAutoResize]="false">
+          <ng-template let-value="value" ngx-datatable-cell-template>
+            <span *ngIf="value" class="state-default state-{{ value }}">
+              {{ value }}
+            </span>
+            <span *ngIf="!value" class="state-PENDING">PENDING</span>
+          </ng-template>
+        </ngx-datatable-column>
+      </ngx-datatable>
+      <ngx-json-viewer *ngSwitchCase="'json'" [json]="workflow.json" fxFill></ngx-json-viewer>
     </section>
   </section>
 </section>

http://git-wip-us.apache.org/repos/asf/helix/blob/b486d784/helix-front/client/app/workflow/workflow-detail/workflow-detail.component.ts
----------------------------------------------------------------------
diff --git a/helix-front/client/app/workflow/workflow-detail/workflow-detail.component.ts
b/helix-front/client/app/workflow/workflow-detail/workflow-detail.component.ts
index 8979d13..f1f04ad 100644
--- a/helix-front/client/app/workflow/workflow-detail/workflow-detail.component.ts
+++ b/helix-front/client/app/workflow/workflow-detail/workflow-detail.component.ts
@@ -2,9 +2,11 @@ import { Component, OnInit } from '@angular/core';
 import { ActivatedRoute } from '@angular/router';
 
 import * as moment from 'moment';
+import * as shape from 'd3-shape';
+import * as _ from 'lodash';
 
 import { Settings } from '../../core/settings';
-import { Workflow } from '../shared/workflow.model';
+import { Workflow, Job } from '../shared/workflow.model';
 import { WorkflowService } from '../shared/workflow.service';
 
 @Component({
@@ -25,7 +27,7 @@ export class WorkflowDetailComponent implements OnInit {
     { prop: 'name', dir: 'asc'}
   ];
   messages = {
-    emptyMessage: 'The queue is empty.',
+    emptyMessage: 'The list is empty.',
     totalMessage: 'total',
     selectedMessage: 'selected'
   };
@@ -58,5 +60,4 @@ export class WorkflowDetailComponent implements OnInit {
     const row = selected[0];
     // this.table.rowDetail.toggleExpandRow(row);
   }
-
 }

http://git-wip-us.apache.org/repos/asf/helix/blob/b486d784/helix-front/client/app/workflow/workflow.module.ts
----------------------------------------------------------------------
diff --git a/helix-front/client/app/workflow/workflow.module.ts b/helix-front/client/app/workflow/workflow.module.ts
index e218928..7ff9868 100644
--- a/helix-front/client/app/workflow/workflow.module.ts
+++ b/helix-front/client/app/workflow/workflow.module.ts
@@ -2,24 +2,30 @@ import { NgModule } from '@angular/core';
 import { CommonModule } from '@angular/common';
 
 import { NgxDatatableModule } from '@swimlane/ngx-datatable';
+import { NgxChartsModule } from '@swimlane/ngx-charts';
+import { NgxGraphModule } from '@swimlane/ngx-graph';
 
 import { WorkflowListComponent } from './workflow-list/workflow-list.component';
 import { WorkflowService } from './shared/workflow.service';
 import { SharedModule } from '../shared/shared.module';
 import { WorkflowDetailComponent } from './workflow-detail/workflow-detail.component';
+import { WorkflowDagComponent } from './workflow-dag/workflow-dag.component';
 
 @NgModule({
   imports: [
     CommonModule,
     SharedModule,
-    NgxDatatableModule
+    NgxDatatableModule,
+    NgxChartsModule,
+    NgxGraphModule
   ],
   providers: [
     WorkflowService
   ],
   declarations: [
     WorkflowListComponent,
-    WorkflowDetailComponent
+    WorkflowDetailComponent,
+    WorkflowDagComponent
   ]
 })
 export class WorkflowModule { }

http://git-wip-us.apache.org/repos/asf/helix/blob/b486d784/helix-front/package.json
----------------------------------------------------------------------
diff --git a/helix-front/package.json b/helix-front/package.json
index bd5fdc9..8c8d1b4 100644
--- a/helix-front/package.json
+++ b/helix-front/package.json
@@ -33,10 +33,13 @@
     "@angular/platform-browser": "^5.1.1",
     "@angular/platform-browser-dynamic": "^5.1.1",
     "@angular/router": "^5.1.1",
+    "@swimlane/ngx-charts": "^7.0.1",
     "@swimlane/ngx-datatable": "^11.1.7",
+    "@swimlane/ngx-graph": "^4.0.2",
     "angulartics2": "^2.2.1",
     "body-parser": "^1.17.2",
     "core-js": "^2.4.1",
+    "d3-shape": "^1.2.0",
     "dotenv": "^4.0.0",
     "express": "^4.15.3",
     "express-session": "^1.15.6",


Mime
View raw message