From d8dc8913ac5d1e6c9654217481ba9fc76d0b909b Mon Sep 17 00:00:00 2001
From: Engin Kayraklioglu <ekayrakli@gmail.com>
Date: Fri, 28 Jul 2017 13:17:04 -0400
Subject: [PATCH 1/4] Add graph500.chpl

---
 benchmark/graph500/graph500.chpl | 487 +++++++++++++++++++++++++++++++
 1 file changed, 487 insertions(+)
 create mode 100644 benchmark/graph500/graph500.chpl

diff --git a/benchmark/graph500/graph500.chpl b/benchmark/graph500/graph500.chpl
new file mode 100644
index 0000000..cc5bd63
--- /dev/null
+++ b/benchmark/graph500/graph500.chpl
@@ -0,0 +1,487 @@
+
+use Random;
+use BlockDist;
+use LayoutCSR;
+use Time;
+use SyncList;
+use Barrier;
+
+var r = new RandomStream(int, seed = 13);
+
+/**
+ * Generates (0, 1, 2, 3) with probabilities (0.57, 0.19, 0.19, 0.05)
+ */
+proc g500_generate_quartile(){
+  const DENOM = 100;
+  const (A_NUM,B_NUM,C_NUM,D_NUM) = (57,19,19,5);
+
+  var rand_num = (abs((r.getNext())%DENOM):int);
+
+  if rand_num < A_NUM then return 0;
+  rand_num -= A_NUM;
+  if rand_num < B_NUM then return 1;
+  rand_num -= B_NUM;
+  if rand_num < C_NUM then return 2;
+  return 3; 
+}
+
+iter g500_vpair_generator(verticeIDs: [] int, numVertices: int, 
+    edgefactor = 16, prng_seed=13){
+  var r = new RandomStream(int, seed=prng_seed); 
+  var u,v,q: int;
+  var quartile_size = numVertices/2;
+
+  const numEdges = numVertices*edgefactor;
+  for edge in 1..numEdges {
+    (u,v) = (0,0);
+    var nv_temp = numVertices;
+    while nv_temp > 1 {
+      q = g500_generate_quartile();            
+      nv_temp /= 2;
+      u += nv_temp * (q/2);
+      v += nv_temp * (q%2);
+    }
+
+    yield (verticeIDs[u], verticeIDs[v]);
+  }
+}
+
+proc createGraph500(scale: int, edgefactor=16, prng_seed=13){
+  const numVertices = 1<<scale;
+  const numEdges = numVertices*edgefactor;
+
+  var verticeIDs: [0..#numVertices] int;
+
+  for i in verticeIDs.domain {
+    verticeIDs[i] = i;
+  }
+
+  /*scrambleVertices(verticeIDs, numVertices);*/
+
+  const space = {0..#numVertices, 0..#numVertices};
+  const denseDom = space dmapped Block(space,
+      sparseLayoutType=CSR);
+  var sparseDom: sparse subdomain(denseDom);
+
+  const indexBufDom = {0..#2*numVertices*edgefactor};
+  var indexBuf: [indexBufDom] 2*int;
+
+  var count = 0;
+  for e in g500_vpair_generator(verticeIDs, numVertices, edgefactor,
+      prng_seed) {
+    indexBuf[count] = e;
+    indexBuf[count+1] = (e[2],e[1]);
+    count += 2;
+  }
+
+  sparseDom += indexBuf;
+  return sparseDom;
+}
+
+proc scrambleVertices(ref verticeIDs: [] int, numVertices: int){
+  var j: int;
+  for i in 0..#numVertices by -1 {
+    j = (abs(r.getNext())%(i+1)):int;
+
+    var tmp = verticeIDs[i];
+    verticeIDs[i] = verticeIDs[j];
+    verticeIDs[j] = tmp;
+  }
+}
+
+config const scale = 5;
+config const edgefactor = 16;
+config const debug = false;
+config const detailed = false;
+config const validation = true;
+config const numSampleKeys = 64;
+config const debugKey = -1;
+
+
+proc numActualNeighbors(g, v) {
+  var count = 0;
+  for x in g.dimIter(2, v) {
+    count+=1;
+  }
+  return count;
+}
+
+proc getBFSParentArray(g, key: int, ref parents: [] int){
+
+  var step = 0;
+
+  parents[key] = key;
+
+  // you can assume queues to be a "work queue":
+  // distributed/unbounded/concurrent-safe enqueue-dequeue
+  var l0 = new SyncList(int);
+  var l1 = new SyncList(int);
+
+  inline proc produceQueue {
+    if step%2 == 0 then return l0;
+    else return l1;
+  }
+
+  inline proc consumeQueue {
+    if step%2 == 1 then return l0;
+    else return l1;
+  }
+
+  inline proc uglyClearWorkAround() {
+    while true {
+      var (x,exists) = consumeQueue.dequeue();
+      if !exists then break;
+    }
+  }
+
+  //parent[i] = j means j is an immediate parent of i in breadth first
+  //iteration that starts from the root.
+
+  consumeQueue.enqueue(key);
+
+  var b = new Barrier(numLocales*here.maxTaskPar);
+  //as long as there vertices to be visited
+  while consumeQueue.data.size != 0 {  
+    coforall l in Locales with (ref step) do on l {
+      coforall t in 0..#here.maxTaskPar with (ref step) {
+        while true {
+          var (exists, v) = consumeQueue.dequeue();
+          if exists {
+            for n in g.dimIter(2,v) { // assume a serial `neighbors` iterator
+              /*writeln("\t", v, ",", n);*/
+
+              // note that this benign race condition is allowed. but you can
+              // assume otherwise, if that makes things easier
+
+              // than the enqueue should be enclosed in a CAS conditional on
+              // parent[n]
+              if parents[n] == -1 { // doesn't have a parent yet (unvisited)
+                parents[n] = v; // "visit"
+                produceQueue.enqueue(n); // add to the "to-be-visited" queue
+              }
+            }
+          }
+          else {
+            break;
+          }
+        }
+        //proceed to the next level
+        b.barrier();
+        if l.id==0 && t==0 {
+          uglyClearWorkAround();
+          /*writeln("Stepping ", consumeQueue.data.size, " ",*/
+                               /*produceQueue.data.size);*/
+          step += 1;
+        }
+      }
+    }
+  }
+}
+
+proc getBFSStats(parents: [] int, g) {
+  var traversedEdges = 0, traversedNodes = 0;
+
+  for i in 0..#(1<<scale) {
+    if parents[i] != -1 { 
+      traversedNodes += 1;
+      traversedEdges += numActualNeighbors(g, i);
+    }
+  }
+
+
+  return (traversedNodes, traversedEdges/2);
+}
+
+proc validate(g, key: int, parents: [] int){
+
+  var passed$: sync bool = true;
+  //nonobligatory checks
+
+  //if root is the only one whose parent is itself
+  if key != parents[key] {
+    writeln("Parent of root is not root");
+    return false;
+  }
+
+  forall i in 0..#(1<<scale) {
+    if parents[i] != -1 { //tree edge
+      if i == parents[i] && i != key {
+        writeln("Nonroot vertex " + i + " has itself as parent");
+        passed$.writeFF(false);
+      }
+    }
+  }
+
+  if !passed$.readFF() then return false;
+
+  //TODO cycle detection
+
+
+  //each tree edge connects vertices whose BFS levels differ by exactly one
+  forall i in 0..#(1<<scale) {
+    if i != key {
+      if parents[i] != -1 { //tree edge
+        if abs(getLevel(i) - getLevel(parents[i])) != 1 {
+          writeln("Condition 2 fails" + i + " " + parents[i] +
+              " " + getLevel(i) + " " + getLevel(parents[i]));
+          passed$.writeFF(false);
+        }
+      }
+    }
+  }
+
+  if !passed$.readFF() then return false;
+
+  //every edge in the input list has vertices with levels 
+  //that differ by at most one or that both are not in the BFS tree,
+  for e in g {
+    //neither in BFS tree
+    if getLevel(e[1])==-1 && getLevel(e[2])==-1 then
+      continue;
+
+    if abs(getLevel(e[1]) - getLevel(e[2])) > 1 {
+      writeln("Condition 3 fails " + e[1] + " " + e[2] + " " + 
+          getLevel(e[1]) + " " + getLevel(e[2]));
+      /*writeln("Parents");*/
+      /*[i in parents.domain] writeln(i, " ", parents[i]);*/
+      /*writeln("Topology");*/
+      /*g.print();*/
+      /*writeln("Schedule");*/
+      /*[i in vertexSpace] writeln(i, " ", g.bfIterator.scheduled_nb[i]);*/
+      /*writeln("HAT Status");*/
+      /*[l in g.bfs_state.levels] l.printStatus();*/
+      return false;
+    }
+  }
+
+  //the BFS tree spans an entire connected component's vertices, and
+
+
+  //a node and its parent are joined by an edge of the original graph.
+
+  // FIXME this check is not easy do to lack of checking if an index
+  // exists in a sparseblockdom
+  /*forall i in 0..#(1<<scale) {*/
+    /*if parents[i] != -1 && i != key{ //tree edge*/
+      /*if !(sparseDomContains(g,(i, parents[i]))) { // replacement for isEdge*/
+        /*writeln("Condition 5 fails. No edge btw " + */
+            /*i + " " + parents[i]);*/
+        /*passed$.writeFF(false);*/
+      /*}*/
+    /*}*/
+  /*}*/
+
+  if !passed$.readFF() then return false;
+
+  return passed$.readFF();
+
+  proc getLevel(v: int, root: int = key){
+
+    if v == root then return 0;
+
+    var lvl = 1;
+    var curParent = parents[v];
+    /*writeln("Checking " + v + " with parent " + curParent);*/
+
+    if curParent == -1 then //not in tree
+      return -1; //something wrong with the tree
+
+    while curParent != root {
+      curParent = parents[curParent];
+      lvl += 1;
+      if curParent == -1 then //parent is not in tree(error)
+        return -2;
+    }
+    return lvl;
+  }
+}
+
+proc main(){
+
+  var t = new Timer();
+  var startTime = 0.0, stopTime = 0.0;
+
+  //kernel 1
+  writeln("Creating input graph");
+  t.start();
+  var g = createGraph500(scale=scale, edgefactor=edgefactor);
+  t.stop();
+  writeln("Graph created successfully");
+
+  var constructionTime = t.elapsed();
+  t.clear();
+
+  /*if debug {*/
+  /*writeln("Topology:");*/
+  /*g.print();*/
+
+
+  /*writeln("BF order:");*/
+  /*for v in g.parBfIterator_parent_vertex() {*/
+  /*writeln(v);*/
+  /*}*/
+  /*}*/
+
+  //sample 64 search keys
+  var sampleKeys: [0..#numSampleKeys] int;
+  var r = new RandomStream(int, seed=13);
+  var keysCreated = 0;
+  for i in 0..#(1<<scale) { //blergh
+    var key = (abs(r.getNext())%(1<<scale)):int;
+
+    //check if vertex is trivial
+    if numActualNeighbors(g, key) == 0 then continue;
+
+    sampleKeys[keysCreated] = key;
+    keysCreated += 1;
+
+    if keysCreated == numSampleKeys then
+      break;
+  }
+
+  //kernel 2 - timed
+  var parents: [0..#(1<<scale)] int = -1;
+
+  var traversedEdges: [0..#keysCreated] int;
+  var times: [0..#keysCreated] real;
+  var teps: [0..#keysCreated] real;
+
+  if debugKey != -1 {
+    writeln("Starting");
+    t.start();        
+    getBFSParentArray(g, debugKey, parents);
+    t.stop();
+
+    if debug then [i in parents.domain] writeln(i, " ", parents[i]);
+
+    if validation then write(if validate(g, debugKey, parents) then "Success\t\t" else "FAIL\t\t");
+
+    /*writeln("Parents");*/
+    /*[i in parents.domain] writeln(i, " ", parents[i]);*/
+    /*writeln("Topology");*/
+    /*g.print();*/
+    /*writeln("Schedule");*/
+    /*[i in vertexSpace] writeln(i, " ", g.bfs_state.scheduled$[i].read());*/
+  }
+  else {
+    writef("\n\n%3s %10s %10s %10s %10s %15s %10s\n", "#", "Vertice ID", "|e|", "|v|",
+        "time", "TEPS", "Valid");
+    for i in 0..#keysCreated {
+      parents = -1;
+
+      writef("%3i %10i ", i, sampleKeys[i]);
+
+      t.start();        
+      getBFSParentArray(g, sampleKeys[i], parents);
+      t.stop();
+
+
+      var (tn, te) = getBFSStats(parents, g);
+
+      writef("%10i %10i ", te, tn);
+      writef("%10.06r %15.6er", t.elapsed(), te/t.elapsed());
+
+      /*writeln("Parents: \n", parents);*/
+      if validation then writef("%10s", 
+          if validate(g, sampleKeys[i], parents) 
+          then "Success" 
+          else "FAIL");
+      else
+        writef("%10s", "N/A");
+
+      /*if detailed then write(getNumAloneVertices(g) + "\n"); else writeln();*/
+      writeln();
+
+      times[i] = t.elapsed();
+      traversedEdges[i] = te;
+      teps[i] = te/t.elapsed();
+      t.clear();
+    }
+
+    //calculate and print out stats
+    writeln("\n\n********** REPORT **********");
+
+    //general stuff
+    writef("%20s:\t%i\n", "Scale:", scale);
+    writef("%20s:\t%i\n", "Edgefactor:", edgefactor);
+    writef("%20s:\t%i\n", "NBFS:", keysCreated);
+    writef("%20s:\t%r\n", "Construction time:", constructionTime);
+
+    //performance
+    var minTime = min reduce times;
+    var firstQTime = times[keysCreated/4];
+    var medianTime = times[keysCreated/2];
+    var thirdQTime = times[keysCreated/4*3];
+    var maxTime = max reduce times;
+
+    var meanTime = (+ reduce times)/keysCreated;
+    var varTime = (+ reduce ([t in times] (t-meanTime)*(t-meanTime)))/keysCreated;
+    var stdTime = sqrt(varTime);
+
+    writeln();
+    writef("%20s:\t%r\n", "Min Time:", minTime);
+    writef("%20s:\t%r\n", "Q1 Time:", firstQTime);
+    writef("%20s:\t%r\n", "Median Time:", medianTime);
+    writef("%20s:\t%r\n", "Q3 Time:", thirdQTime);
+    writef("%20s:\t%r\n", "Max Time:", maxTime);
+
+    writef("%20s:\t%r\n", "Mean Time:", meanTime);
+    writef("%20s:\t%r\n", "Var Time:", varTime);
+    writef("%20s:\t%dr\n", "StdDev Time:", stdTime);
+
+    var minNEdges = min reduce traversedEdges;
+    var firstQNEdges = traversedEdges[keysCreated/4];
+    var medianNEdges = traversedEdges[keysCreated/2];
+    var thirdQNEdges = traversedEdges[keysCreated/4*3];
+    var maxNEdges = max reduce traversedEdges;
+
+    var meanNEdges = (+ reduce traversedEdges)/keysCreated;
+    var varNEdges = (+ reduce ([t in traversedEdges]
+          1.0*(t-meanNEdges)*(t-meanNEdges)))/keysCreated;
+    var stdNEdges = sqrt(varNEdges);
+
+    writeln();
+    writef("%20s:\t%r\n", "Min NEdges:", minNEdges);
+    writef("%20s:\t%r\n", "Q1 NEdges:", firstQNEdges);
+    writef("%20s:\t%r\n", "Median NEdges:", medianNEdges);
+    writef("%20s:\t%r\n", "Q3 NEdges:", thirdQNEdges);
+    writef("%20s:\t%r\n", "Max NEdges:", maxNEdges);
+
+    writef("%20s:\t%r\n", "Mean NEdges:", meanNEdges);
+    writef("%20s:\t%r\n", "Var NEdges:", varNEdges);
+    writef("%20s:\t%r\n", "StdDev NEdges:", stdNEdges);
+
+    var minTeps = min reduce teps;
+    var firstQTeps = teps[keysCreated/4];
+    var medianTeps = teps[keysCreated/2];
+    var thirdQTeps = teps[keysCreated/4*3];
+    var maxTeps = max reduce teps;
+
+    var harmonicMeanTeps = keysCreated / (+ reduce (1/teps) );
+    var harmonicStdDevTeps = sqrt( (+ reduce (1/teps-1/harmonicMeanTeps)) /
+        (keysCreated-1) ) * harmonicMeanTeps*harmonicMeanTeps;
+
+    writeln();
+    writef("%20s:\t%r\n", "Min Teps:", minTeps);
+    writef("%20s:\t%r\n", "Q1 Teps:", firstQTeps);
+    writef("%20s:\t%r\n", "Median Teps:", medianTeps);
+    writef("%20s:\t%r\n", "Q3 Teps:", thirdQTeps);
+    writef("%20s:\t%r\n", "Max Teps:", maxTeps);
+
+    writef("%20s:\t%r\n", "Harm. Mean Teps:", harmonicMeanTeps);
+    writef("%20s:\t%r\n", "Harm. StdDev Teps:", harmonicStdDevTeps);
+
+  }
+
+}
+
+iter SparseBlockDom.dimIter(param dim, idx) {
+  var targetLocRow = dist.targetLocsIdx((idx, whole.dim(2).low));
+  /*writeln("dimIter idx: ", idx, " targetLocRow ", targetLocRow);*/
+
+  for l in dist.targetLocales.domain.dim(2) {
+    for i in locDoms[(targetLocRow[1], l)].mySparseBlock.dimIter(2, idx) {
+      yield i;
+    }
+  }
+}

From 3d78841044b7d909f96e670fbd1a1e631b1b2b05 Mon Sep 17 00:00:00 2001
From: Engin Kayraklioglu <ekayrakli@gmail.com>
Date: Fri, 28 Jul 2017 13:20:22 -0400
Subject: [PATCH 2/4] Add skeleton build system for graph500

---
 benchmark/graph500/Makefile       | 5 +++++
 benchmark/graph500/bin/.gitignore | 2 ++
 2 files changed, 7 insertions(+)
 create mode 100644 benchmark/graph500/Makefile
 create mode 100644 benchmark/graph500/bin/.gitignore

diff --git a/benchmark/graph500/Makefile b/benchmark/graph500/Makefile
new file mode 100644
index 0000000..3049e1e
--- /dev/null
+++ b/benchmark/graph500/Makefile
@@ -0,0 +1,5 @@
+graph500: graph500.chpl
+	chpl -M../../queues -M../../queues/local -o bin/$@ $^
+
+clean:
+	rm bin/graph500*
diff --git a/benchmark/graph500/bin/.gitignore b/benchmark/graph500/bin/.gitignore
new file mode 100644
index 0000000..d6b7ef3
--- /dev/null
+++ b/benchmark/graph500/bin/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore

From e6a58f1e3026ae5c7b840bf728f967d278511cd1 Mon Sep 17 00:00:00 2001
From: Louis Jenkins <LouisJenkinsCS@hotmail.com>
Date: Fri, 18 Aug 2017 14:40:39 -0400
Subject: [PATCH 3/4] Graph500 unit testing

---
 benchmark/graph500/Makefile       |  2 +-
 benchmark/graph500/graph500.chpl  | 95 +++++++++++++------------------
 collections/DistributedDeque.chpl | 19 ++++++-
 collections/SynchronizedList.chpl | 12 ++++
 4 files changed, 69 insertions(+), 59 deletions(-)

diff --git a/benchmark/graph500/Makefile b/benchmark/graph500/Makefile
index 3049e1e..f646821 100644
--- a/benchmark/graph500/Makefile
+++ b/benchmark/graph500/Makefile
@@ -1,5 +1,5 @@
 graph500: graph500.chpl
-	chpl -M../../queues -M../../queues/local -o bin/$@ $^
+	chpl -M../../collections -o bin/$@ $^
 
 clean:
 	rm bin/graph500*
diff --git a/benchmark/graph500/graph500.chpl b/benchmark/graph500/graph500.chpl
index cc5bd63..dd3edb3 100644
--- a/benchmark/graph500/graph500.chpl
+++ b/benchmark/graph500/graph500.chpl
@@ -3,7 +3,7 @@ use Random;
 use BlockDist;
 use LayoutCSR;
 use Time;
-use SyncList;
+use SynchronizedList;
 use Barrier;
 
 var r = new RandomStream(int, seed = 13);
@@ -22,12 +22,12 @@ proc g500_generate_quartile(){
   if rand_num < B_NUM then return 1;
   rand_num -= B_NUM;
   if rand_num < C_NUM then return 2;
-  return 3; 
+  return 3;
 }
 
-iter g500_vpair_generator(verticeIDs: [] int, numVertices: int, 
+iter g500_vpair_generator(verticeIDs: [] int, numVertices: int,
     edgefactor = 16, prng_seed=13){
-  var r = new RandomStream(int, seed=prng_seed); 
+  var r = new RandomStream(int, seed=prng_seed);
   var u,v,q: int;
   var quartile_size = numVertices/2;
 
@@ -36,7 +36,7 @@ iter g500_vpair_generator(verticeIDs: [] int, numVertices: int,
     (u,v) = (0,0);
     var nv_temp = numVertices;
     while nv_temp > 1 {
-      q = g500_generate_quartile();            
+      q = g500_generate_quartile();
       nv_temp /= 2;
       u += nv_temp * (q/2);
       v += nv_temp * (q%2);
@@ -114,8 +114,16 @@ proc getBFSParentArray(g, key: int, ref parents: [] int){
 
   // you can assume queues to be a "work queue":
   // distributed/unbounded/concurrent-safe enqueue-dequeue
-  var l0 = new SyncList(int);
-  var l1 = new SyncList(int);
+  var l0 : Collection(int);
+  var l1 : Collection(int);
+
+  if isBag {
+    l0 = new DistributedBag(int);
+    l1 = new DistributedBag(int);
+  } else if isDeque {
+    l0 = new DistributedDeque(int);
+    l1 = new DistributedDeque(int);
+  }
 
   inline proc produceQueue {
     if step%2 == 0 then return l0;
@@ -127,54 +135,31 @@ proc getBFSParentArray(g, key: int, ref parents: [] int){
     else return l1;
   }
 
-  inline proc uglyClearWorkAround() {
-    while true {
-      var (x,exists) = consumeQueue.dequeue();
-      if !exists then break;
-    }
-  }
-
   //parent[i] = j means j is an immediate parent of i in breadth first
   //iteration that starts from the root.
 
-  consumeQueue.enqueue(key);
-
+  consumeQueue.add(key);
+  produceQueue.freeze();
   var b = new Barrier(numLocales*here.maxTaskPar);
   //as long as there vertices to be visited
-  while consumeQueue.data.size != 0 {  
-    coforall l in Locales with (ref step) do on l {
-      coforall t in 0..#here.maxTaskPar with (ref step) {
-        while true {
-          var (exists, v) = consumeQueue.dequeue();
-          if exists {
-            for n in g.dimIter(2,v) { // assume a serial `neighbors` iterator
-              /*writeln("\t", v, ",", n);*/
-
-              // note that this benign race condition is allowed. but you can
-              // assume otherwise, if that makes things easier
-
-              // than the enqueue should be enclosed in a CAS conditional on
-              // parent[n]
-              if parents[n] == -1 { // doesn't have a parent yet (unvisited)
-                parents[n] = v; // "visit"
-                produceQueue.enqueue(n); // add to the "to-be-visited" queue
-              }
-            }
-          }
-          else {
-            break;
-          }
-        }
-        //proceed to the next level
-        b.barrier();
-        if l.id==0 && t==0 {
-          uglyClearWorkAround();
-          /*writeln("Stepping ", consumeQueue.data.size, " ",*/
-                               /*produceQueue.data.size);*/
-          step += 1;
+  while !consumeQueue.isEmpty() {
+    forall v in consumeQueue {
+      for n in g.dimIter(2,v) { // assume a serial `neighbors` iterator
+        /*writeln("\t", v, ",", n);*/
+
+        // note that this benign race condition is allowed. but you can
+        // assume otherwise, if that makes things easier
+
+        // than the enqueue should be enclosed in a CAS conditional on
+        // parent[n]
+        if parents[n] == -1 { // doesn't have a parent yet (unvisited)
+          parents[n] = v; // "visit"
+          produceQueue.add(n); // add to the "to-be-visited" queue
         }
       }
     }
+    consumeQueue.clear();
+    step += 1;
   }
 }
 
@@ -182,7 +167,7 @@ proc getBFSStats(parents: [] int, g) {
   var traversedEdges = 0, traversedNodes = 0;
 
   for i in 0..#(1<<scale) {
-    if parents[i] != -1 { 
+    if parents[i] != -1 {
       traversedNodes += 1;
       traversedEdges += numActualNeighbors(g, i);
     }
@@ -232,7 +217,7 @@ proc validate(g, key: int, parents: [] int){
 
   if !passed$.readFF() then return false;
 
-  //every edge in the input list has vertices with levels 
+  //every edge in the input list has vertices with levels
   //that differ by at most one or that both are not in the BFS tree,
   for e in g {
     //neither in BFS tree
@@ -240,7 +225,7 @@ proc validate(g, key: int, parents: [] int){
       continue;
 
     if abs(getLevel(e[1]) - getLevel(e[2])) > 1 {
-      writeln("Condition 3 fails " + e[1] + " " + e[2] + " " + 
+      writeln("Condition 3 fails " + e[1] + " " + e[2] + " " +
           getLevel(e[1]) + " " + getLevel(e[2]));
       /*writeln("Parents");*/
       /*[i in parents.domain] writeln(i, " ", parents[i]);*/
@@ -348,7 +333,7 @@ proc main(){
 
   if debugKey != -1 {
     writeln("Starting");
-    t.start();        
+    t.start();
     getBFSParentArray(g, debugKey, parents);
     t.stop();
 
@@ -371,7 +356,7 @@ proc main(){
 
       writef("%3i %10i ", i, sampleKeys[i]);
 
-      t.start();        
+      t.start();
       getBFSParentArray(g, sampleKeys[i], parents);
       t.stop();
 
@@ -382,9 +367,9 @@ proc main(){
       writef("%10.06r %15.6er", t.elapsed(), te/t.elapsed());
 
       /*writeln("Parents: \n", parents);*/
-      if validation then writef("%10s", 
-          if validate(g, sampleKeys[i], parents) 
-          then "Success" 
+      if validation then writef("%10s",
+          if validate(g, sampleKeys[i], parents)
+          then "Success"
           else "FAIL");
       else
         writef("%10s", "N/A");
diff --git a/collections/DistributedDeque.chpl b/collections/DistributedDeque.chpl
index 9324d5a..93034d1 100644
--- a/collections/DistributedDeque.chpl
+++ b/collections/DistributedDeque.chpl
@@ -9,7 +9,12 @@ use Collection;
   of checking whether you may proceed with an operation, that is wait-free under
   most cases, lock-free in the worst case, which guarantees scalability. In the
   barrier, we perform freeze checks (if DEQUE_NO_FREEZE is not enabled), and bounds
-  checking (if a capacity is given). 
+  checking (if a capacity is given). Deque operations are separated into two
+  levels: global and local. At a global level, we use simple fetchAdd and fetchSub
+  counters to denote which local deque we apply our operation to, and at a local level
+  we use an unrolled linked list which further has its own (non-atomic) counter.
+  By maintaining an
+
 */
 
 /*
@@ -791,7 +796,11 @@ class DistributedDeque : Collection {
     }
   }
 
-  iter FIFO() {
+  /*
+    Iterates over the deque in First-In-First-Out order, from front to back. The
+    deque must be frozen or it will result in a halt. This operation is sequential.
+  */
+  iter FIFO() : eltType {
     if !isFrozen() {
       halt("Ordered iteration requires the queue to be frozen.");
     }
@@ -850,7 +859,11 @@ class DistributedDeque : Collection {
     }
   }
 
-  iter LIFO() {
+  /*
+    Iterates over the deque in Last-In-First-Out order, from back to front. The
+    deque must be frozen or it will result in a halt. This operation is sequential.
+  */
+  iter LIFO() : eltType {
     if !isFrozen() {
       halt("Ordered iteration requires the queue to be frozen.");
     }
diff --git a/collections/SynchronizedList.chpl b/collections/SynchronizedList.chpl
index af0ff50..f2ca026 100644
--- a/collections/SynchronizedList.chpl
+++ b/collections/SynchronizedList.chpl
@@ -10,6 +10,7 @@ class SynchronizedList : Collection {
   var head : SynchronizedListNode(eltType);
   var tail : SynchronizedListNode(eltType);
   var lock$ : sync bool;
+  var _size : int;
 
   proc add(elt : eltType) : bool {
     on this {
@@ -22,8 +23,11 @@ class SynchronizedList : Collection {
         tail.next = node;
         tail = node;
       }
+      this._size += 1;
       lock$;
     }
+
+
     return true;
   }
 
@@ -44,9 +48,17 @@ class SynchronizedList : Collection {
           head = head.next;
           delete tmp;
         }
+
+        this._size -= 1;
       }
+
       lock$;
     }
+
     return (hasElem, elem);
   }
+
+  proc size() : int {
+    return _size;
+  }
 }

From 0253a8673498afdb197c840069effa1f39e91004 Mon Sep 17 00:00:00 2001
From: Louis Jenkins <LouisJenkinsCS@hotmail.com>
Date: Fri, 18 Aug 2017 16:28:12 -0400
Subject: [PATCH 4/4] Segfault with graph500

---
 benchmark/graph500/graph500.chpl |  23 +-
 bt.svg                           | 689 +++++++++++++++++++++++++++++++
 2 files changed, 698 insertions(+), 14 deletions(-)
 create mode 100644 bt.svg

diff --git a/benchmark/graph500/graph500.chpl b/benchmark/graph500/graph500.chpl
index dd3edb3..286a055 100644
--- a/benchmark/graph500/graph500.chpl
+++ b/benchmark/graph500/graph500.chpl
@@ -3,7 +3,8 @@ use Random;
 use BlockDist;
 use LayoutCSR;
 use Time;
-use SynchronizedList;
+use DistributedBag;
+use DistributedDeque;
 use Barrier;
 
 var r = new RandomStream(int, seed = 13);
@@ -106,24 +107,17 @@ proc numActualNeighbors(g, v) {
   return count;
 }
 
+config param isBag = false;
+
 proc getBFSParentArray(g, key: int, ref parents: [] int){
 
   var step = 0;
 
   parents[key] = key;
 
-  // you can assume queues to be a "work queue":
-  // distributed/unbounded/concurrent-safe enqueue-dequeue
-  var l0 : Collection(int);
-  var l1 : Collection(int);
-
-  if isBag {
-    l0 = new DistributedBag(int);
-    l1 = new DistributedBag(int);
-  } else if isDeque {
-    l0 = new DistributedDeque(int);
-    l1 = new DistributedDeque(int);
-  }
+  // Our data structures...
+  var l0 = (if isBag then new DistributedBag(int) else new DistributedDeque(int));
+  var l1 = (if isBag then new DistributedBag(int) else new DistributedDeque(int));
 
   inline proc produceQueue {
     if step%2 == 0 then return l0;
@@ -139,10 +133,10 @@ proc getBFSParentArray(g, key: int, ref parents: [] int){
   //iteration that starts from the root.
 
   consumeQueue.add(key);
-  produceQueue.freeze();
   var b = new Barrier(numLocales*here.maxTaskPar);
   //as long as there vertices to be visited
   while !consumeQueue.isEmpty() {
+    consumeQueue.freeze();
     forall v in consumeQueue {
       for n in g.dimIter(2,v) { // assume a serial `neighbors` iterator
         /*writeln("\t", v, ",", n);*/
@@ -158,6 +152,7 @@ proc getBFSParentArray(g, key: int, ref parents: [] int){
         }
       }
     }
+    consumeQueue.unfreeze();
     consumeQueue.clear();
     step += 1;
   }
diff --git a/bt.svg b/bt.svg
new file mode 100644
index 0000000..0bf4777
--- /dev/null
+++ b/bt.svg
@@ -0,0 +1,689 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.28.0 (20140926.2003)
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="3120pt" height="1394pt"
+ viewBox="0.00 0.00 3120.00 1394.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph1" class="graph" transform="scale(1 1) rotate(0) translate(4 1390)">
+<title>G</title>
+<polygon fill="white" stroke="white" points="-4,5 -4,-1390 3117,-1390 3117,5 -4,5"/>
+<!-- 0 -->
+<g id="node1" class="node"><title>0</title>
+<polygon fill="#aaaaaa" stroke="black" points="2166,-1360.6 2166,-1385.4 2220,-1385.4 2220,-1360.6 2166,-1360.6"/>
+<text text-anchor="middle" x="2192.95" y="-1368.8" font-family="Times,serif" font-size="14.00" fill="#ffffff">/</text>
+</g>
+<!-- 315179265 -->
+<g id="node2" class="node"><title>315179265</title>
+<polygon fill="#f1c71c" stroke="black" points="1229.87,-1280.6 1229.87,-1305.4 1350.13,-1305.4 1350.13,-1280.6 1229.87,-1280.6"/>
+<text text-anchor="middle" x="1290" y="-1288.8" font-family="Times,serif" font-size="14.00" fill="#000000">_start@start.S:122</text>
+</g>
+<!-- 0&#45;&gt;315179265 -->
+<g id="edge2" class="edge"><title>0&#45;&gt;315179265</title>
+<path fill="none" stroke="black" d="M2165.87,-1370.4C2098.07,-1366.33 1915.77,-1354.99 1764.02,-1342 1620.44,-1329.71 1452.75,-1311.92 1360.26,-1301.8"/>
+<polygon fill="black" stroke="black" points="1360.5,-1298.31 1350.18,-1300.7 1359.74,-1305.27 1360.5,-1298.31"/>
+<text text-anchor="middle" x="1782.49" y="-1328.8" font-family="Times,serif" font-size="14.00">[0&#45;31]</text>
+</g>
+<!-- &#45;691397763 -->
+<g id="node10" class="node"><title>&#45;691397763</title>
+<polygon fill="#d55555" stroke="black" points="1450.11,-1280.6 1450.11,-1305.4 1611.89,-1305.4 1611.89,-1280.6 1450.11,-1280.6"/>
+<text text-anchor="middle" x="1531" y="-1288.8" font-family="Times,serif" font-size="14.00" fill="#000000">[empty]@0xffffffffffffffff</text>
+</g>
+<!-- 0&#45;&gt;&#45;691397763 -->
+<g id="edge18" class="edge"><title>0&#45;&gt;&#45;691397763</title>
+<path fill="none" stroke="black" d="M2165.81,-1368.8C2074.37,-1358.02 1775.07,-1322.76 1622.38,-1304.77"/>
+<polygon fill="black" stroke="black" points="1622.43,-1301.25 1612.09,-1303.55 1621.61,-1308.2 1622.43,-1301.25"/>
+<text text-anchor="middle" x="1948.24" y="-1328.8" font-family="Times,serif" font-size="14.00">[0&#45;28,30]</text>
+</g>
+<!-- &#45;2093759535 -->
+<g id="node27" class="node"><title>&#45;2093759535</title>
+<polygon fill="#f1c71c" stroke="black" points="2087.02,-1280.6 2087.02,-1305.4 2298.98,-1305.4 2298.98,-1280.6 2087.02,-1280.6"/>
+<text text-anchor="middle" x="2193" y="-1288.8" font-family="Times,serif" font-size="14.00" fill="#000000">start_thread@pthread_create.c:309</text>
+</g>
+<!-- 0&#45;&gt;&#45;2093759535 -->
+<g id="edge52" class="edge"><title>0&#45;&gt;&#45;2093759535</title>
+<path fill="none" stroke="black" d="M2193,-1360.49C2193,-1348.74 2193,-1330.21 2193,-1315.7"/>
+<polygon fill="black" stroke="black" points="2196.5,-1315.56 2193,-1305.56 2189.5,-1315.56 2196.5,-1315.56"/>
+<text text-anchor="middle" x="2210.49" y="-1328.8" font-family="Times,serif" font-size="14.00">[0&#45;31]</text>
+</g>
+<!-- &#45;2134501904 -->
+<g id="node43" class="node"><title>&#45;2134501904</title>
+<polygon fill="#c71c71" stroke="black" points="2345.98,-1280.6 2345.98,-1305.4 2458.02,-1305.4 2458.02,-1280.6 2345.98,-1280.6"/>
+<text text-anchor="middle" x="2402" y="-1288.8" font-family="Times,serif" font-size="14.00" fill="#000000">[Fault Summary]</text>
+</g>
+<!-- 0&#45;&gt;&#45;2134501904 -->
+<g id="edge84" class="edge"><title>0&#45;&gt;&#45;2134501904</title>
+<path fill="none" stroke="black" d="M2220.32,-1361.8C2256.38,-1348.35 2320.14,-1324.55 2361.72,-1309.03"/>
+<polygon fill="black" stroke="black" points="2363.13,-1312.24 2371.27,-1305.47 2360.68,-1305.68 2363.13,-1312.24"/>
+<text text-anchor="middle" x="2389.32" y="-1328.8" font-family="Times,serif" font-size="14.00">[0,5,7,11,13&#45;15,19&#45;28,30]</text>
+</g>
+<!-- 762131901 -->
+<g id="node50" class="node"><title>762131901</title>
+<polygon fill="#b8e38e" stroke="black" points="2505.23,-1280.6 2505.23,-1305.4 2844.77,-1305.4 2844.77,-1280.6 2505.23,-1280.6"/>
+<text text-anchor="middle" x="2675" y="-1288.8" font-family="Times,serif" font-size="14.00" fill="#000000">qthread_master$$CFE_id_58403e1c_qlib@qthread.c:536</text>
+</g>
+<!-- 0&#45;&gt;762131901 -->
+<g id="edge98" class="edge"><title>0&#45;&gt;762131901</title>
+<path fill="none" stroke="black" d="M2220.17,-1370.04C2269.28,-1366.27 2375.99,-1357.03 2465,-1342 2518.77,-1332.92 2579.41,-1318.61 2621.66,-1307.98"/>
+<polygon fill="black" stroke="black" points="2622.62,-1311.34 2631.45,-1305.49 2620.9,-1304.56 2622.62,-1311.34"/>
+<text text-anchor="middle" x="2569.49" y="-1328.8" font-family="Times,serif" font-size="14.00">[1&#45;31]</text>
+</g>
+<!-- &#45;347316729 -->
+<g id="node3" class="node"><title>&#45;347316729</title>
+<polygon fill="#f1c71c" stroke="black" points="510.826,-1200.6 510.826,-1225.4 725.174,-1225.4 725.174,-1200.6 510.826,-1200.6"/>
+<text text-anchor="middle" x="618" y="-1208.8" font-family="Times,serif" font-size="14.00" fill="#000000">__libc_start_main@libc&#45;start.c:285</text>
+</g>
+<!-- 315179265&#45;&gt;&#45;347316729 -->
+<g id="edge4" class="edge"><title>315179265&#45;&gt;&#45;347316729</title>
+<path fill="none" stroke="black" d="M1229.77,-1285.01C1116.36,-1271.85 869.67,-1243.21 726.607,-1226.61"/>
+<polygon fill="black" stroke="black" points="726.825,-1223.11 716.488,-1225.43 726.018,-1230.06 726.825,-1223.11"/>
+<text text-anchor="middle" x="1032.49" y="-1248.8" font-family="Times,serif" font-size="14.00">[0&#45;31]</text>
+</g>
+<!-- &#45;1919329781 -->
+<g id="node4" class="node"><title>&#45;1919329781</title>
+<polygon fill="#e38e38" stroke="black" points="302.755,-1120.6 302.755,-1145.4 415.245,-1145.4 415.245,-1120.6 302.755,-1120.6"/>
+<text text-anchor="middle" x="359" y="-1128.8" font-family="Times,serif" font-size="14.00" fill="#000000">main@main.c:32</text>
+</g>
+<!-- &#45;347316729&#45;&gt;&#45;1919329781 -->
+<g id="edge6" class="edge"><title>&#45;347316729&#45;&gt;&#45;1919329781</title>
+<path fill="none" stroke="black" d="M580.062,-1200.57C534.09,-1186.73 456.598,-1163.39 406.646,-1148.35"/>
+<polygon fill="black" stroke="black" points="407.624,-1144.99 397.04,-1145.46 405.606,-1151.69 407.624,-1144.99"/>
+<text text-anchor="middle" x="520.161" y="-1168.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- &#45;1919329777 -->
+<g id="node45" class="node"><title>&#45;1919329777</title>
+<polygon fill="#b8e38e" stroke="black" points="561.755,-1120.6 561.755,-1145.4 674.245,-1145.4 674.245,-1120.6 561.755,-1120.6"/>
+<text text-anchor="middle" x="618" y="-1128.8" font-family="Times,serif" font-size="14.00" fill="#000000">main@main.c:36</text>
+</g>
+<!-- &#45;347316729&#45;&gt;&#45;1919329777 -->
+<g id="edge88" class="edge"><title>&#45;347316729&#45;&gt;&#45;1919329777</title>
+<path fill="none" stroke="black" d="M618,-1200.49C618,-1188.74 618,-1170.21 618,-1155.7"/>
+<polygon fill="black" stroke="black" points="621.5,-1155.56 618,-1145.56 614.5,-1155.56 621.5,-1155.56"/>
+<text text-anchor="middle" x="635.491" y="-1168.8" font-family="Times,serif" font-size="14.00">[1&#45;31]</text>
+</g>
+<!-- 1032997230 -->
+<g id="node5" class="node"><title>1032997230</title>
+<polygon fill="#e38e38" stroke="black" points="105.004,-1040.6 105.004,-1065.4 358.996,-1065.4 358.996,-1040.6 105.004,-1040.6"/>
+<text text-anchor="middle" x="232" y="-1048.8" font-family="Times,serif" font-size="14.00" fill="#000000">chpl_task_callMain@tasks&#45;qthreads.c:857</text>
+</g>
+<!-- &#45;1919329781&#45;&gt;1032997230 -->
+<g id="edge8" class="edge"><title>&#45;1919329781&#45;&gt;1032997230</title>
+<path fill="none" stroke="black" d="M340.261,-1120.49C318.969,-1107.41 284.018,-1085.95 259.606,-1070.95"/>
+<polygon fill="black" stroke="black" points="261.169,-1067.81 250.816,-1065.56 257.506,-1073.77 261.169,-1067.81"/>
+<text text-anchor="middle" x="315.161" y="-1088.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- 498383936 -->
+<g id="node6" class="node"><title>498383936</title>
+<polygon fill="#e38e38" stroke="black" points="-0.2504,-960.6 -0.2504,-985.4 458.25,-985.4 458.25,-960.6 -0.2504,-960.6"/>
+<text text-anchor="middle" x="229" y="-968.8" font-family="Times,serif" font-size="14.00" fill="#000000">qthread_syncvar_blocker_func$$CFE_id_7902d5df_e51184f1@syncvar.c:333</text>
+</g>
+<!-- 1032997230&#45;&gt;498383936 -->
+<g id="edge10" class="edge"><title>1032997230&#45;&gt;498383936</title>
+<path fill="none" stroke="black" d="M231.557,-1040.49C231.105,-1028.74 230.393,-1010.21 229.834,-995.696"/>
+<polygon fill="black" stroke="black" points="233.326,-995.414 229.444,-985.556 226.331,-995.684 233.326,-995.414"/>
+<text text-anchor="middle" x="239.161" y="-1008.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- 395734293 -->
+<g id="node7" class="node"><title>395734293</title>
+<polygon fill="#e38e38" stroke="black" points="117.149,-880.6 117.149,-905.4 340.851,-905.4 340.851,-880.6 117.149,-880.6"/>
+<text text-anchor="middle" x="229" y="-888.8" font-family="Times,serif" font-size="14.00" fill="#000000">__lll_lock_elision@elision&#45;lock.c:94</text>
+</g>
+<!-- 498383936&#45;&gt;395734293 -->
+<g id="edge12" class="edge"><title>498383936&#45;&gt;395734293</title>
+<path fill="none" stroke="black" d="M229,-960.491C229,-948.739 229,-930.212 229,-915.696"/>
+<polygon fill="black" stroke="black" points="232.5,-915.556 229,-905.556 225.5,-915.556 232.5,-915.556"/>
+<text text-anchor="middle" x="237.161" y="-928.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- &#45;236289326 -->
+<g id="node8" class="node"><title>&#45;236289326</title>
+<polygon fill="#e38e38" stroke="black" points="129.392,-800.6 129.392,-825.4 328.608,-825.4 328.608,-800.6 129.392,-800.6"/>
+<text text-anchor="middle" x="229" y="-808.8" font-family="Times,serif" font-size="14.00" fill="#000000">__lll_lock_elision@0x203125c9</text>
+</g>
+<!-- 395734293&#45;&gt;&#45;236289326 -->
+<g id="edge14" class="edge"><title>395734293&#45;&gt;&#45;236289326</title>
+<path fill="none" stroke="black" d="M229,-880.491C229,-868.739 229,-850.212 229,-835.696"/>
+<polygon fill="black" stroke="black" points="232.5,-835.556 229,-825.556 225.5,-835.556 232.5,-835.556"/>
+<text text-anchor="middle" x="237.161" y="-848.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- 314791733 -->
+<g id="node9" class="node"><title>314791733</title>
+<polygon fill="#e38e38" stroke="black" points="116.371,-720.6 116.371,-745.4 341.629,-745.4 341.629,-720.6 116.371,-720.6"/>
+<text text-anchor="middle" x="229" y="-728.8" font-family="Times,serif" font-size="14.00" fill="#000000">__lll_lock_wait@lowlevellock.S:135</text>
+</g>
+<!-- &#45;236289326&#45;&gt;314791733 -->
+<g id="edge16" class="edge"><title>&#45;236289326&#45;&gt;314791733</title>
+<path fill="none" stroke="black" d="M229,-800.491C229,-788.739 229,-770.212 229,-755.696"/>
+<polygon fill="black" stroke="black" points="232.5,-755.556 229,-745.556 225.5,-755.556 232.5,-755.556"/>
+<text text-anchor="middle" x="237.161" y="-768.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- 1539693960 -->
+<g id="node11" class="node"><title>1539693960</title>
+<polygon fill="#e38e38" stroke="black" points="935.456,-1200.6 935.456,-1225.4 1290.54,-1225.4 1290.54,-1200.6 935.456,-1200.6"/>
+<text text-anchor="middle" x="1113" y="-1208.8" font-family="Times,serif" font-size="14.00" fill="#000000">qthread_wrapper$$CFE_id_58403e1c_qlib@qthread.c:2292</text>
+</g>
+<!-- &#45;691397763&#45;&gt;1539693960 -->
+<g id="edge20" class="edge"><title>&#45;691397763&#45;&gt;1539693960</title>
+<path fill="none" stroke="black" d="M1469.77,-1280.57C1393.88,-1266.41 1264.76,-1242.32 1184.42,-1227.33"/>
+<polygon fill="black" stroke="black" points="1184.86,-1223.85 1174.39,-1225.46 1183.58,-1230.73 1184.86,-1223.85"/>
+<text text-anchor="middle" x="1368.16" y="-1248.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- 1539693967 -->
+<g id="node31" class="node"><title>1539693967</title>
+<polygon fill="#d55555" stroke="black" points="1353.46,-1200.6 1353.46,-1225.4 1708.54,-1225.4 1708.54,-1200.6 1353.46,-1200.6"/>
+<text text-anchor="middle" x="1531" y="-1208.8" font-family="Times,serif" font-size="14.00" fill="#000000">qthread_wrapper$$CFE_id_58403e1c_qlib@qthread.c:2299</text>
+</g>
+<!-- &#45;691397763&#45;&gt;1539693967 -->
+<g id="edge60" class="edge"><title>&#45;691397763&#45;&gt;1539693967</title>
+<path fill="none" stroke="black" d="M1531,-1280.49C1531,-1268.74 1531,-1250.21 1531,-1235.7"/>
+<polygon fill="black" stroke="black" points="1534.5,-1235.56 1531,-1225.56 1527.5,-1235.56 1534.5,-1235.56"/>
+<text text-anchor="middle" x="1557.24" y="-1248.8" font-family="Times,serif" font-size="14.00">[0&#45;28,30]</text>
+</g>
+<!-- 909284907 -->
+<g id="node12" class="node"><title>909284907</title>
+<polygon fill="#e38e38" stroke="black" points="840.959,-1120.6 840.959,-1145.4 1245.04,-1145.4 1245.04,-1120.6 840.959,-1120.6"/>
+<text text-anchor="middle" x="1043" y="-1128.8" font-family="Times,serif" font-size="14.00" fill="#000000">main_wrapper$$CFE_id_70d80183_ed8f0e2b@tasks&#45;qthreads.c:791</text>
+</g>
+<!-- 1539693960&#45;&gt;909284907 -->
+<g id="edge22" class="edge"><title>1539693960&#45;&gt;909284907</title>
+<path fill="none" stroke="black" d="M1102.67,-1200.49C1091.5,-1188.05 1073.52,-1168.01 1060.21,-1153.18"/>
+<polygon fill="black" stroke="black" points="1062.66,-1150.66 1053.37,-1145.56 1057.45,-1155.34 1062.66,-1150.66"/>
+<text text-anchor="middle" x="1093.16" y="-1168.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- 982260061 -->
+<g id="node13" class="node"><title>982260061</title>
+<polygon fill="#e38e38" stroke="black" points="881.217,-1040.6 881.217,-1065.4 1108.78,-1065.4 1108.78,-1040.6 881.217,-1040.6"/>
+<text text-anchor="middle" x="995" y="-1048.8" font-family="Times,serif" font-size="14.00" fill="#000000">chpl_executable_init@chpl&#45;init.c:268</text>
+</g>
+<!-- 909284907&#45;&gt;982260061 -->
+<g id="edge24" class="edge"><title>909284907&#45;&gt;982260061</title>
+<path fill="none" stroke="black" d="M1035.92,-1120.49C1028.47,-1108.39 1016.61,-1089.12 1007.57,-1074.43"/>
+<polygon fill="black" stroke="black" points="1010.33,-1072.24 1002.11,-1065.56 1004.37,-1075.91 1010.33,-1072.24"/>
+<text text-anchor="middle" x="1032.16" y="-1088.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- 1830505643 -->
+<g id="node14" class="node"><title>1830505643</title>
+<polygon fill="#e38e38" stroke="black" points="884.71,-960.6 884.71,-985.4 1105.29,-985.4 1105.29,-960.6 884.71,-960.6"/>
+<text text-anchor="middle" x="995" y="-968.8" font-family="Times,serif" font-size="14.00" fill="#000000">chpl_gen_main@graph500.chpl:279</text>
+</g>
+<!-- 982260061&#45;&gt;1830505643 -->
+<g id="edge26" class="edge"><title>982260061&#45;&gt;1830505643</title>
+<path fill="none" stroke="black" d="M995,-1040.49C995,-1028.74 995,-1010.21 995,-995.696"/>
+<polygon fill="black" stroke="black" points="998.5,-995.556 995,-985.556 991.5,-995.556 998.5,-995.556"/>
+<text text-anchor="middle" x="1003.16" y="-1008.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- 598495046 -->
+<g id="node15" class="node"><title>598495046</title>
+<polygon fill="#e38e38" stroke="black" points="774.117,-880.6 774.117,-905.4 1175.88,-905.4 1175.88,-880.6 774.117,-880.6"/>
+<text text-anchor="middle" x="975" y="-888.8" font-family="Times,serif" font-size="14.00" fill="#000000">chpl_user_main$$CFE_id_f9ee5c55_9b0c5384@graph500.chpl:287</text>
+</g>
+<!-- 1830505643&#45;&gt;598495046 -->
+<g id="edge28" class="edge"><title>1830505643&#45;&gt;598495046</title>
+<path fill="none" stroke="black" d="M992.049,-960.491C989.006,-948.624 984.192,-929.848 980.454,-915.27"/>
+<polygon fill="black" stroke="black" points="983.837,-914.374 977.963,-905.556 977.057,-916.112 983.837,-914.374"/>
+<text text-anchor="middle" x="995.161" y="-928.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- &#45;297919505 -->
+<g id="node16" class="node"><title>&#45;297919505</title>
+<polygon fill="#e38e38" stroke="black" points="741.079,-800.6 741.079,-825.4 1166.92,-825.4 1166.92,-800.6 741.079,-800.6"/>
+<text text-anchor="middle" x="954" y="-808.8" font-family="Times,serif" font-size="14.00" fill="#000000">createGraph500_chpl$$CFE_id_f9ee5c55_9b0c5384@graph500.chpl:78</text>
+</g>
+<!-- 598495046&#45;&gt;&#45;297919505 -->
+<g id="edge30" class="edge"><title>598495046&#45;&gt;&#45;297919505</title>
+<path fill="none" stroke="black" d="M971.901,-880.491C968.706,-868.624 963.651,-849.848 959.727,-835.27"/>
+<polygon fill="black" stroke="black" points="963.091,-834.303 957.111,-825.556 956.331,-836.122 963.091,-834.303"/>
+<text text-anchor="middle" x="975.161" y="-848.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- 825781990 -->
+<g id="node17" class="node"><title>825781990</title>
+<polygon fill="#e38e38" stroke="black" points="652.167,-720.6 652.167,-745.4 1169.83,-745.4 1169.83,-720.6 652.167,-720.6"/>
+<text text-anchor="middle" x="911" y="-728.8" font-family="Times,serif" font-size="14.00" fill="#000000">chpl___PLUS__ASSIGN_$$CFE_id_f9ee5c55_9b0c5384@ChapelDistribution.chpl:606</text>
+</g>
+<!-- &#45;297919505&#45;&gt;825781990 -->
+<g id="edge32" class="edge"><title>&#45;297919505&#45;&gt;825781990</title>
+<path fill="none" stroke="black" d="M947.655,-800.491C940.986,-788.394 930.359,-769.116 922.26,-754.426"/>
+<polygon fill="black" stroke="black" points="925.264,-752.624 917.371,-745.556 919.134,-756.004 925.264,-752.624"/>
+<text text-anchor="middle" x="945.161" y="-768.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- 1725954841 -->
+<g id="node18" class="node"><title>1725954841</title>
+<polygon fill="#e38e38" stroke="black" points="693.77,-640.6 693.77,-665.4 1128.23,-665.4 1128.23,-640.6 693.77,-640.6"/>
+<text text-anchor="middle" x="911" y="-648.8" font-family="Times,serif" font-size="14.00" fill="#000000">dsiBulkAdd$$CFE_id_f9ee5c55_9b0c5384@ChapelDistribution.chpl:407</text>
+</g>
+<!-- 825781990&#45;&gt;1725954841 -->
+<g id="edge34" class="edge"><title>825781990&#45;&gt;1725954841</title>
+<path fill="none" stroke="black" d="M911,-720.491C911,-708.739 911,-690.212 911,-675.696"/>
+<polygon fill="black" stroke="black" points="914.5,-675.556 911,-665.556 907.5,-675.556 914.5,-675.556"/>
+<text text-anchor="middle" x="919.161" y="-688.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- 1273159494 -->
+<g id="node19" class="node"><title>1273159494</title>
+<polygon fill="#e38e38" stroke="black" points="675.116,-560.6 675.116,-585.4 1146.88,-585.4 1146.88,-560.6 675.116,-560.6"/>
+<text text-anchor="middle" x="911" y="-568.8" font-family="Times,serif" font-size="14.00" fill="#000000">bulkAdd_help_chpl3$$CFE_id_f9ee5c55_9b0c5384@SparseBlockDist.chpl:163</text>
+</g>
+<!-- 1725954841&#45;&gt;1273159494 -->
+<g id="edge36" class="edge"><title>1725954841&#45;&gt;1273159494</title>
+<path fill="none" stroke="black" d="M911,-640.491C911,-628.739 911,-610.212 911,-595.696"/>
+<polygon fill="black" stroke="black" points="914.5,-595.556 911,-585.556 907.5,-595.556 914.5,-595.556"/>
+<text text-anchor="middle" x="919.161" y="-608.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- &#45;2024049826 -->
+<g id="node20" class="node"><title>&#45;2024049826</title>
+<polygon fill="#e38e38" stroke="black" points="700.391,-480.6 700.391,-505.4 1121.61,-505.4 1121.61,-480.6 700.391,-480.6"/>
+<text text-anchor="middle" x="911" y="-488.8" font-family="Times,serif" font-size="14.00" fill="#000000">_waitEndCount3$$CFE_id_f9ee5c55_9b0c5384@ChapelBase.chpl:964</text>
+</g>
+<!-- 1273159494&#45;&gt;&#45;2024049826 -->
+<g id="edge38" class="edge"><title>1273159494&#45;&gt;&#45;2024049826</title>
+<path fill="none" stroke="black" d="M911,-560.491C911,-548.739 911,-530.212 911,-515.696"/>
+<polygon fill="black" stroke="black" points="914.5,-515.556 911,-505.556 907.5,-515.556 914.5,-515.556"/>
+<text text-anchor="middle" x="919.161" y="-528.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- &#45;1119827012 -->
+<g id="node21" class="node"><title>&#45;1119827012</title>
+<polygon fill="#e38e38" stroke="black" points="706.227,-400.6 706.227,-425.4 1115.77,-425.4 1115.77,-400.6 706.227,-400.6"/>
+<text text-anchor="middle" x="911" y="-408.8" font-family="Times,serif" font-size="14.00" fill="#000000">on_fn136$$CFE_id_f9ee5c55_9b0c5384@NetworkAtomics.chpl:170</text>
+</g>
+<!-- &#45;2024049826&#45;&gt;&#45;1119827012 -->
+<g id="edge40" class="edge"><title>&#45;2024049826&#45;&gt;&#45;1119827012</title>
+<path fill="none" stroke="black" d="M911,-480.491C911,-468.739 911,-450.212 911,-435.696"/>
+<polygon fill="black" stroke="black" points="914.5,-435.556 911,-425.556 907.5,-435.556 914.5,-435.556"/>
+<text text-anchor="middle" x="919.161" y="-448.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- &#45;1782324773 -->
+<g id="node22" class="node"><title>&#45;1782324773</title>
+<polygon fill="#e38e38" stroke="black" points="711.271,-320.6 711.271,-345.4 1110.73,-345.4 1110.73,-320.6 711.271,-320.6"/>
+<text text-anchor="middle" x="911" y="-328.8" font-family="Times,serif" font-size="14.00" fill="#000000">do_remote_get$$CFE_id_f94beb93_9960313b@comm&#45;ugni.c:4059</text>
+</g>
+<!-- &#45;1119827012&#45;&gt;&#45;1782324773 -->
+<g id="edge42" class="edge"><title>&#45;1119827012&#45;&gt;&#45;1782324773</title>
+<path fill="none" stroke="black" d="M911,-400.491C911,-388.739 911,-370.212 911,-355.696"/>
+<polygon fill="black" stroke="black" points="914.5,-355.556 911,-345.556 907.5,-355.556 914.5,-355.556"/>
+<text text-anchor="middle" x="919.161" y="-368.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- &#45;1034640353 -->
+<g id="node23" class="node"><title>&#45;1034640353</title>
+<polygon fill="#e38e38" stroke="black" points="698.05,-240.6 698.05,-265.4 1123.95,-265.4 1123.95,-240.6 698.05,-240.6"/>
+<text text-anchor="middle" x="911" y="-248.8" font-family="Times,serif" font-size="14.00" fill="#000000">post_fma_and_wait$$CFE_id_f94beb93_9960313b@comm&#45;ugni.c:6171</text>
+</g>
+<!-- &#45;1782324773&#45;&gt;&#45;1034640353 -->
+<g id="edge44" class="edge"><title>&#45;1782324773&#45;&gt;&#45;1034640353</title>
+<path fill="none" stroke="black" d="M911,-320.491C911,-308.739 911,-290.212 911,-275.696"/>
+<polygon fill="black" stroke="black" points="914.5,-275.556 911,-265.556 907.5,-275.556 914.5,-275.556"/>
+<text text-anchor="middle" x="919.161" y="-288.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- 625971348 -->
+<g id="node24" class="node"><title>625971348</title>
+<polygon fill="#e38e38" stroke="black" points="727.208,-160.6 727.208,-185.4 1094.79,-185.4 1094.79,-160.6 727.208,-160.6"/>
+<text text-anchor="middle" x="911" y="-168.8" font-family="Times,serif" font-size="14.00" fill="#000000">post_fma$$CFE_id_f94beb93_9960313b@comm&#45;ugni.c:6147</text>
+</g>
+<!-- &#45;1034640353&#45;&gt;625971348 -->
+<g id="edge46" class="edge"><title>&#45;1034640353&#45;&gt;625971348</title>
+<path fill="none" stroke="black" d="M911,-240.491C911,-228.739 911,-210.212 911,-195.696"/>
+<polygon fill="black" stroke="black" points="914.5,-195.556 911,-185.556 907.5,-195.556 914.5,-195.556"/>
+<text text-anchor="middle" x="919.161" y="-208.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- &#45;907068018 -->
+<g id="node25" class="node"><title>&#45;907068018</title>
+<polygon fill="#e38e38" stroke="black" points="695.718,-80.6 695.718,-105.4 1126.28,-105.4 1126.28,-80.6 695.718,-80.6"/>
+<text text-anchor="middle" x="911" y="-88.8" font-family="Times,serif" font-size="14.00" fill="#000000">acquire_comm_dom$$CFE_id_f94beb93_9960313b@comm&#45;ugni.c:5928</text>
+</g>
+<!-- 625971348&#45;&gt;&#45;907068018 -->
+<g id="edge48" class="edge"><title>625971348&#45;&gt;&#45;907068018</title>
+<path fill="none" stroke="black" d="M911,-160.491C911,-148.739 911,-130.212 911,-115.696"/>
+<polygon fill="black" stroke="black" points="914.5,-115.556 911,-105.556 907.5,-115.556 914.5,-115.556"/>
+<text text-anchor="middle" x="919.161" y="-128.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- &#45;274508891 -->
+<g id="node26" class="node"><title>&#45;274508891</title>
+<polygon fill="#e38e38" stroke="black" points="677.44,-0.6 677.44,-25.4 1144.56,-25.4 1144.56,-0.6 677.44,-0.6"/>
+<text text-anchor="middle" x="911" y="-8.8" font-family="Times,serif" font-size="14.00" fill="#000000">atomic_load_explicit_bool$$CFE_id_f94beb93_9960313b@chpl&#45;atomics.h:321</text>
+</g>
+<!-- &#45;907068018&#45;&gt;&#45;274508891 -->
+<g id="edge50" class="edge"><title>&#45;907068018&#45;&gt;&#45;274508891</title>
+<path fill="none" stroke="black" d="M911,-80.4908C911,-68.7392 911,-50.2119 911,-35.6961"/>
+<polygon fill="black" stroke="black" points="914.5,-35.5564 911,-25.5564 907.5,-35.5564 914.5,-35.5564"/>
+<text text-anchor="middle" x="919.161" y="-48.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- &#45;852437746 -->
+<g id="node28" class="node"><title>&#45;852437746</title>
+<polygon fill="#f1c71c" stroke="black" points="1850.23,-1200.6 1850.23,-1225.4 2189.77,-1225.4 2189.77,-1200.6 1850.23,-1200.6"/>
+<text text-anchor="middle" x="2020" y="-1208.8" font-family="Times,serif" font-size="14.00" fill="#000000">qthread_master$$CFE_id_58403e1c_qlib@qthread.c:536</text>
+</g>
+<!-- &#45;2093759535&#45;&gt;&#45;852437746 -->
+<g id="edge54" class="edge"><title>&#45;2093759535&#45;&gt;&#45;852437746</title>
+<path fill="none" stroke="black" d="M2167.47,-1280.49C2137.62,-1267.03 2088.05,-1244.68 2054.71,-1229.65"/>
+<polygon fill="black" stroke="black" points="2055.9,-1226.35 2045.35,-1225.43 2053.03,-1232.73 2055.9,-1226.35"/>
+<text text-anchor="middle" x="2139.49" y="-1248.8" font-family="Times,serif" font-size="14.00">[0&#45;31]</text>
+</g>
+<!-- 1425139660 -->
+<g id="node40" class="node"><title>1425139660</title>
+<polygon fill="#f1c71c" stroke="black" points="2207.68,-1200.6 2207.68,-1225.4 2648.32,-1225.4 2648.32,-1200.6 2207.68,-1200.6"/>
+<text text-anchor="middle" x="2428" y="-1208.8" font-family="Times,serif" font-size="14.00" fill="#000000">comm_task_wrapper$$CFE_id_70d80183_ed8f0e2b@tasks&#45;qthreads.c:831</text>
+</g>
+<!-- &#45;2093759535&#45;&gt;1425139660 -->
+<g id="edge78" class="edge"><title>&#45;2093759535&#45;&gt;1425139660</title>
+<path fill="none" stroke="black" d="M2227.42,-1280.57C2268.87,-1266.82 2338.57,-1243.68 2383.92,-1228.63"/>
+<polygon fill="black" stroke="black" points="2385.1,-1231.93 2393.49,-1225.46 2382.89,-1225.28 2385.1,-1231.93"/>
+<text text-anchor="middle" x="2349.49" y="-1248.8" font-family="Times,serif" font-size="14.00">[0&#45;31]</text>
+</g>
+<!-- &#45;1396505020 -->
+<g id="node29" class="node"><title>&#45;1396505020</title>
+<polygon fill="#f1c71c" stroke="black" points="1903.47,-1120.6 1903.47,-1145.4 2230.53,-1145.4 2230.53,-1120.6 1903.47,-1120.6"/>
+<text text-anchor="middle" x="2067" y="-1128.8" font-family="Times,serif" font-size="14.00" fill="#000000">qt_scheduler_get_thread@nemesis_threadqueues.c:397</text>
+</g>
+<!-- &#45;852437746&#45;&gt;&#45;1396505020 -->
+<g id="edge56" class="edge"><title>&#45;852437746&#45;&gt;&#45;1396505020</title>
+<path fill="none" stroke="black" d="M2026.94,-1200.49C2034.22,-1188.39 2045.84,-1169.12 2054.69,-1154.43"/>
+<polygon fill="black" stroke="black" points="2057.87,-1155.93 2060.04,-1145.56 2051.88,-1152.32 2057.87,-1155.93"/>
+<text text-anchor="middle" x="2065.49" y="-1168.8" font-family="Times,serif" font-size="14.00">[0&#45;31]</text>
+</g>
+<!-- 859176990 -->
+<g id="node30" class="node"><title>859176990</title>
+<polygon fill="#f1c71c" stroke="black" points="2078.28,-1040.6 2078.28,-1065.4 2435.72,-1065.4 2435.72,-1040.6 2078.28,-1040.6"/>
+<text text-anchor="middle" x="2257" y="-1048.8" font-family="Times,serif" font-size="14.00" fill="#000000">__pthread_cond_timedwait@pthread_cond_timedwait.S:238</text>
+</g>
+<!-- &#45;1396505020&#45;&gt;859176990 -->
+<g id="edge58" class="edge"><title>&#45;1396505020&#45;&gt;859176990</title>
+<path fill="none" stroke="black" d="M2095.04,-1120.49C2128.11,-1106.91 2183.2,-1084.29 2219.82,-1069.26"/>
+<polygon fill="black" stroke="black" points="2221.24,-1072.46 2229.16,-1065.43 2218.58,-1065.99 2221.24,-1072.46"/>
+<text text-anchor="middle" x="2197.49" y="-1088.8" font-family="Times,serif" font-size="14.00">[0&#45;31]</text>
+</g>
+<!-- 1615102860 -->
+<g id="node32" class="node"><title>1615102860</title>
+<polygon fill="#d55555" stroke="black" points="1324.69,-1120.6 1324.69,-1145.4 1737.31,-1145.4 1737.31,-1120.6 1324.69,-1120.6"/>
+<text text-anchor="middle" x="1531" y="-1128.8" font-family="Times,serif" font-size="14.00" fill="#000000">chapel_wrapper$$CFE_id_70d80183_ed8f0e2b@tasks&#45;qthreads.c:812</text>
+</g>
+<!-- 1539693967&#45;&gt;1615102860 -->
+<g id="edge62" class="edge"><title>1539693967&#45;&gt;1615102860</title>
+<path fill="none" stroke="black" d="M1531,-1200.49C1531,-1188.74 1531,-1170.21 1531,-1155.7"/>
+<polygon fill="black" stroke="black" points="1534.5,-1155.56 1531,-1145.56 1527.5,-1155.56 1534.5,-1155.56"/>
+<text text-anchor="middle" x="1557.24" y="-1168.8" font-family="Times,serif" font-size="14.00">[0&#45;28,30]</text>
+</g>
+<!-- &#45;919839913 -->
+<g id="node33" class="node"><title>&#45;919839913</title>
+<polygon fill="#e38e38" stroke="black" points="1128.35,-1040.6 1128.35,-1065.4 1587.65,-1065.4 1587.65,-1040.6 1128.35,-1040.6"/>
+<text text-anchor="middle" x="1358" y="-1048.8" font-family="Times,serif" font-size="14.00" fill="#000000">wrapon_fn_chpl64$$CFE_id_f9ee5c55_9b0c5384@SparseBlockDist.chpl:163</text>
+</g>
+<!-- 1615102860&#45;&gt;&#45;919839913 -->
+<g id="edge64" class="edge"><title>1615102860&#45;&gt;&#45;919839913</title>
+<path fill="none" stroke="black" d="M1505.47,-1120.49C1475.62,-1107.03 1426.05,-1084.68 1392.71,-1069.65"/>
+<polygon fill="black" stroke="black" points="1393.9,-1066.35 1383.35,-1065.43 1391.03,-1072.73 1393.9,-1066.35"/>
+<text text-anchor="middle" x="1468.16" y="-1088.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- 1940492027 -->
+<g id="node53" class="node"><title>1940492027</title>
+<polygon fill="#aaaaaa" stroke="black" points="1606.07,-1040.6 1606.07,-1065.4 2059.93,-1065.4 2059.93,-1040.6 1606.07,-1040.6"/>
+<text text-anchor="middle" x="1833" y="-1048.8" font-family="Times,serif" font-size="14.00" fill="#000000">fork_call_wrapper_large$$CFE_id_f94beb93_9960313b@comm&#45;ugni.c:2936</text>
+</g>
+<!-- 1615102860&#45;&gt;1940492027 -->
+<g id="edge104" class="edge"><title>1615102860&#45;&gt;1940492027</title>
+<path fill="none" stroke="black" d="M1575.24,-1120.57C1629.29,-1106.61 1720.71,-1083 1778.89,-1067.97"/>
+<polygon fill="black" stroke="black" points="1779.84,-1071.35 1788.64,-1065.46 1778.09,-1064.57 1779.84,-1071.35"/>
+<text text-anchor="middle" x="1736.24" y="-1088.8" font-family="Times,serif" font-size="14.00">[1&#45;28,30]</text>
+</g>
+<!-- 1063170007 -->
+<g id="node34" class="node"><title>1063170007</title>
+<polygon fill="#e38e38" stroke="black" points="1153.34,-960.6 1153.34,-985.4 1584.66,-985.4 1584.66,-960.6 1153.34,-960.6"/>
+<text text-anchor="middle" x="1369" y="-968.8" font-family="Times,serif" font-size="14.00" fill="#000000">on_fn_chpl64$$CFE_id_f9ee5c55_9b0c5384@SparseBlockDist.chpl:164</text>
+</g>
+<!-- &#45;919839913&#45;&gt;1063170007 -->
+<g id="edge66" class="edge"><title>&#45;919839913&#45;&gt;1063170007</title>
+<path fill="none" stroke="black" d="M1359.62,-1040.49C1361.28,-1028.74 1363.89,-1010.21 1365.94,-995.696"/>
+<polygon fill="black" stroke="black" points="1369.44,-995.947 1367.37,-985.556 1362.51,-994.97 1369.44,-995.947"/>
+<text text-anchor="middle" x="1373.16" y="-1008.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- 1890331699 -->
+<g id="node35" class="node"><title>1890331699</title>
+<polygon fill="#e38e38" stroke="black" points="1211.12,-880.6 1211.12,-905.4 1598.88,-905.4 1598.88,-880.6 1211.12,-880.6"/>
+<text text-anchor="middle" x="1405" y="-888.8" font-family="Times,serif" font-size="14.00" fill="#000000">bulkAdd$$CFE_id_f9ee5c55_9b0c5384@ChapelArray.chpl:1336</text>
+</g>
+<!-- 1063170007&#45;&gt;1890331699 -->
+<g id="edge68" class="edge"><title>1063170007&#45;&gt;1890331699</title>
+<path fill="none" stroke="black" d="M1374.31,-960.491C1379.84,-948.509 1388.62,-929.483 1395.38,-914.847"/>
+<polygon fill="black" stroke="black" points="1398.65,-916.103 1399.67,-905.556 1392.3,-913.169 1398.65,-916.103"/>
+<text text-anchor="middle" x="1399.16" y="-928.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- &#45;1669328512 -->
+<g id="node36" class="node"><title>&#45;1669328512</title>
+<polygon fill="#e38e38" stroke="black" points="1185.27,-800.6 1185.27,-825.4 1626.73,-825.4 1626.73,-800.6 1185.27,-800.6"/>
+<text text-anchor="middle" x="1406" y="-808.8" font-family="Times,serif" font-size="14.00" fill="#000000">dsiBulkAdd2$$CFE_id_f9ee5c55_9b0c5384@ChapelDistribution.chpl:410</text>
+</g>
+<!-- 1890331699&#45;&gt;&#45;1669328512 -->
+<g id="edge70" class="edge"><title>1890331699&#45;&gt;&#45;1669328512</title>
+<path fill="none" stroke="black" d="M1405.15,-880.491C1405.3,-868.739 1405.54,-850.212 1405.72,-835.696"/>
+<polygon fill="black" stroke="black" points="1409.22,-835.6 1405.85,-825.556 1402.22,-835.511 1409.22,-835.6"/>
+<text text-anchor="middle" x="1414.16" y="-848.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- 40334548 -->
+<g id="node37" class="node"><title>40334548</title>
+<polygon fill="#e38e38" stroke="black" points="1187.6,-720.6 1187.6,-745.4 1624.4,-745.4 1624.4,-720.6 1187.6,-720.6"/>
+<text text-anchor="middle" x="1406" y="-728.8" font-family="Times,serif" font-size="14.00" fill="#000000">bulkAdd_help_chpl$$CFE_id_f9ee5c55_9b0c5384@LayoutCSR.chpl:300</text>
+</g>
+<!-- &#45;1669328512&#45;&gt;40334548 -->
+<g id="edge72" class="edge"><title>&#45;1669328512&#45;&gt;40334548</title>
+<path fill="none" stroke="black" d="M1406,-800.491C1406,-788.739 1406,-770.212 1406,-755.696"/>
+<polygon fill="black" stroke="black" points="1409.5,-755.556 1406,-745.556 1402.5,-755.556 1409.5,-755.556"/>
+<text text-anchor="middle" x="1414.16" y="-768.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- 963249411 -->
+<g id="node38" class="node"><title>963249411</title>
+<polygon fill="#e38e38" stroke="black" points="1317.68,-640.6 1317.68,-665.4 1494.32,-665.4 1494.32,-640.6 1317.68,-640.6"/>
+<text text-anchor="middle" x="1406" y="-648.8" font-family="Times,serif" font-size="14.00" fill="#000000">__cray_EXP2@0x2058d6cd</text>
+</g>
+<!-- 40334548&#45;&gt;963249411 -->
+<g id="edge74" class="edge"><title>40334548&#45;&gt;963249411</title>
+<path fill="none" stroke="black" d="M1406,-720.491C1406,-708.739 1406,-690.212 1406,-675.696"/>
+<polygon fill="black" stroke="black" points="1409.5,-675.556 1406,-665.556 1402.5,-675.556 1409.5,-675.556"/>
+<text text-anchor="middle" x="1414.16" y="-688.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- &#45;415726767 -->
+<g id="node39" class="node"><title>&#45;415726767</title>
+<polygon fill="#e38e38" stroke="black" points="1299.01,-560.6 1299.01,-585.4 1512.99,-585.4 1512.99,-560.6 1299.01,-560.6"/>
+<text text-anchor="middle" x="1406" y="-568.8" font-family="Times,serif" font-size="14.00" fill="#000000">__cray_EXP2_Z_03@0x2058d70b</text>
+</g>
+<!-- 963249411&#45;&gt;&#45;415726767 -->
+<g id="edge76" class="edge"><title>963249411&#45;&gt;&#45;415726767</title>
+<path fill="none" stroke="black" d="M1406,-640.491C1406,-628.739 1406,-610.212 1406,-595.696"/>
+<polygon fill="black" stroke="black" points="1409.5,-595.556 1406,-585.556 1402.5,-595.556 1409.5,-595.556"/>
+<text text-anchor="middle" x="1414.16" y="-608.8" font-family="Times,serif" font-size="14.00">[0]</text>
+</g>
+<!-- 1511576705 -->
+<g id="node41" class="node"><title>1511576705</title>
+<polygon fill="#f1c71c" stroke="black" points="2323.65,-1120.6 2323.65,-1145.4 2708.35,-1145.4 2708.35,-1120.6 2323.65,-1120.6"/>
+<text text-anchor="middle" x="2516" y="-1128.8" font-family="Times,serif" font-size="14.00" fill="#000000">polling_task$$CFE_id_f94beb93_9960313b@comm&#45;ugni.c:2176</text>
+</g>
+<!-- 1425139660&#45;&gt;1511576705 -->
+<g id="edge80" class="edge"><title>1425139660&#45;&gt;1511576705</title>
+<path fill="none" stroke="black" d="M2440.98,-1200.49C2455.28,-1187.82 2478.47,-1167.26 2495.29,-1152.36"/>
+<polygon fill="black" stroke="black" points="2497.8,-1154.81 2502.96,-1145.56 2493.16,-1149.57 2497.8,-1154.81"/>
+<text text-anchor="middle" x="2497.49" y="-1168.8" font-family="Times,serif" font-size="14.00">[0&#45;31]</text>
+</g>
+<!-- 2020477110 -->
+<g id="node42" class="node"><title>2020477110</title>
+<polygon fill="#f1c71c" stroke="black" points="2454.05,-1040.6 2454.05,-1065.4 2669.95,-1065.4 2669.95,-1040.6 2454.05,-1040.6"/>
+<text text-anchor="middle" x="2562" y="-1048.8" font-family="Times,serif" font-size="14.00" fill="#000000">sched_yield@syscall&#45;template.S:81</text>
+</g>
+<!-- 1511576705&#45;&gt;2020477110 -->
+<g id="edge82" class="edge"><title>1511576705&#45;&gt;2020477110</title>
+<path fill="none" stroke="black" d="M2522.79,-1120.49C2529.92,-1108.39 2541.29,-1089.12 2549.95,-1074.43"/>
+<polygon fill="black" stroke="black" points="2553.12,-1075.95 2555.18,-1065.56 2547.09,-1072.39 2553.12,-1075.95"/>
+<text text-anchor="middle" x="2561.49" y="-1088.8" font-family="Times,serif" font-size="14.00">[0&#45;31]</text>
+</g>
+<!-- 549102563 -->
+<g id="node44" class="node"><title>549102563</title>
+<polygon fill="#c71c71" stroke="black" points="2666.02,-1200.6 2666.02,-1225.4 2751.98,-1225.4 2751.98,-1200.6 2666.02,-1200.6"/>
+<text text-anchor="middle" x="2709" y="-1208.8" font-family="Times,serif" font-size="14.00" fill="#000000">[SIGILL(4)]</text>
+</g>
+<!-- &#45;2134501904&#45;&gt;549102563 -->
+<g id="edge86" class="edge"><title>&#45;2134501904&#45;&gt;549102563</title>
+<path fill="none" stroke="black" d="M2446.97,-1280.57C2502.55,-1266.45 2596.98,-1242.46 2656.02,-1227.46"/>
+<polygon fill="black" stroke="black" points="2657.06,-1230.81 2665.89,-1224.95 2655.34,-1224.02 2657.06,-1230.81"/>
+<text text-anchor="middle" x="2655.32" y="-1248.8" font-family="Times,serif" font-size="14.00">[0,5,7,11,13&#45;15,19&#45;28,30]</text>
+</g>
+<!-- 741169504 -->
+<g id="node46" class="node"><title>741169504</title>
+<polygon fill="#b8e38e" stroke="black" points="378.211,-1040.6 378.211,-1065.4 857.789,-1065.4 857.789,-1040.6 378.211,-1040.6"/>
+<text text-anchor="middle" x="618" y="-1048.8" font-family="Times,serif" font-size="14.00" fill="#000000">chpl_exit_common$$CFE_id_85563794_39134d7b_clone_6119_1@chplexit.c:54</text>
+</g>
+<!-- &#45;1919329777&#45;&gt;741169504 -->
+<g id="edge90" class="edge"><title>&#45;1919329777&#45;&gt;741169504</title>
+<path fill="none" stroke="black" d="M618,-1120.49C618,-1108.74 618,-1090.21 618,-1075.7"/>
+<polygon fill="black" stroke="black" points="621.5,-1075.56 618,-1065.56 614.5,-1075.56 621.5,-1075.56"/>
+<text text-anchor="middle" x="635.491" y="-1088.8" font-family="Times,serif" font-size="14.00">[1&#45;31]</text>
+</g>
+<!-- 988388986 -->
+<g id="node47" class="node"><title>988388986</title>
+<polygon fill="#b8e38e" stroke="black" points="476.217,-960.6 476.217,-985.4 759.783,-985.4 759.783,-960.6 476.217,-960.6"/>
+<text text-anchor="middle" x="618" y="-968.8" font-family="Times,serif" font-size="14.00" fill="#000000">chpl_comm_pre_task_exit@comm&#45;ugni.c:2617</text>
+</g>
+<!-- 741169504&#45;&gt;988388986 -->
+<g id="edge92" class="edge"><title>741169504&#45;&gt;988388986</title>
+<path fill="none" stroke="black" d="M618,-1040.49C618,-1028.74 618,-1010.21 618,-995.696"/>
+<polygon fill="black" stroke="black" points="621.5,-995.556 618,-985.556 614.5,-995.556 621.5,-995.556"/>
+<text text-anchor="middle" x="635.491" y="-1008.8" font-family="Times,serif" font-size="14.00">[1&#45;31]</text>
+</g>
+<!-- &#45;1335818896 -->
+<g id="node48" class="node"><title>&#45;1335818896</title>
+<polygon fill="#b8e38e" stroke="black" points="494.276,-880.6 494.276,-905.4 739.724,-905.4 739.724,-880.6 494.276,-880.6"/>
+<text text-anchor="middle" x="617" y="-888.8" font-family="Times,serif" font-size="14.00" fill="#000000">chpl_comm_barrier@comm&#45;ugni.c:2586</text>
+</g>
+<!-- 988388986&#45;&gt;&#45;1335818896 -->
+<g id="edge94" class="edge"><title>988388986&#45;&gt;&#45;1335818896</title>
+<path fill="none" stroke="black" d="M617.852,-960.491C617.702,-948.739 617.464,-930.212 617.278,-915.696"/>
+<polygon fill="black" stroke="black" points="620.776,-915.511 617.148,-905.556 613.777,-915.6 620.776,-915.511"/>
+<text text-anchor="middle" x="635.491" y="-928.8" font-family="Times,serif" font-size="14.00">[1&#45;31]</text>
+</g>
+<!-- 1819904903 -->
+<g id="node49" class="node"><title>1819904903</title>
+<polygon fill="#b8e38e" stroke="black" points="507.049,-800.6 507.049,-825.4 722.951,-825.4 722.951,-800.6 507.049,-800.6"/>
+<text text-anchor="middle" x="615" y="-808.8" font-family="Times,serif" font-size="14.00" fill="#000000">sched_yield@syscall&#45;template.S:81</text>
+</g>
+<!-- &#45;1335818896&#45;&gt;1819904903 -->
+<g id="edge96" class="edge"><title>&#45;1335818896&#45;&gt;1819904903</title>
+<path fill="none" stroke="black" d="M616.705,-880.491C616.404,-868.739 615.929,-850.212 615.556,-835.696"/>
+<polygon fill="black" stroke="black" points="619.052,-835.463 615.296,-825.556 612.054,-835.643 619.052,-835.463"/>
+<text text-anchor="middle" x="634.491" y="-848.8" font-family="Times,serif" font-size="14.00">[1&#45;31]</text>
+</g>
+<!-- &#45;533771405 -->
+<g id="node51" class="node"><title>&#45;533771405</title>
+<polygon fill="#b8e38e" stroke="black" points="2770.47,-1200.6 2770.47,-1225.4 3097.53,-1225.4 3097.53,-1200.6 2770.47,-1200.6"/>
+<text text-anchor="middle" x="2934" y="-1208.8" font-family="Times,serif" font-size="14.00" fill="#000000">qt_scheduler_get_thread@nemesis_threadqueues.c:397</text>
+</g>
+<!-- 762131901&#45;&gt;&#45;533771405 -->
+<g id="edge100" class="edge"><title>762131901&#45;&gt;&#45;533771405</title>
+<path fill="none" stroke="black" d="M2787.51,-1280.56C2810.93,-1276.23 2835.12,-1270.24 2857,-1262 2876.53,-1254.64 2896.47,-1242.04 2911.17,-1231.6"/>
+<polygon fill="black" stroke="black" points="2913.58,-1234.17 2919.59,-1225.44 2909.45,-1228.52 2913.58,-1234.17"/>
+<text text-anchor="middle" x="2910.49" y="-1248.8" font-family="Times,serif" font-size="14.00">[1&#45;31]</text>
+</g>
+<!-- 1277266351 -->
+<g id="node52" class="node"><title>1277266351</title>
+<polygon fill="#b8e38e" stroke="black" points="2755.28,-1120.6 2755.28,-1145.4 3112.72,-1145.4 3112.72,-1120.6 2755.28,-1120.6"/>
+<text text-anchor="middle" x="2934" y="-1128.8" font-family="Times,serif" font-size="14.00" fill="#000000">__pthread_cond_timedwait@pthread_cond_timedwait.S:238</text>
+</g>
+<!-- &#45;533771405&#45;&gt;1277266351 -->
+<g id="edge102" class="edge"><title>&#45;533771405&#45;&gt;1277266351</title>
+<path fill="none" stroke="black" d="M2934,-1200.49C2934,-1188.74 2934,-1170.21 2934,-1155.7"/>
+<polygon fill="black" stroke="black" points="2937.5,-1155.56 2934,-1145.56 2930.5,-1155.56 2937.5,-1155.56"/>
+<text text-anchor="middle" x="2951.49" y="-1168.8" font-family="Times,serif" font-size="14.00">[1&#45;31]</text>
+</g>
+<!-- &#45;1223359704 -->
+<g id="node54" class="node"><title>&#45;1223359704</title>
+<polygon fill="#aaaaaa" stroke="black" points="1605.35,-960.6 1605.35,-985.4 2064.65,-985.4 2064.65,-960.6 1605.35,-960.6"/>
+<text text-anchor="middle" x="1835" y="-968.8" font-family="Times,serif" font-size="14.00" fill="#000000">wrapon_fn_chpl64$$CFE_id_f9ee5c55_9b0c5384@SparseBlockDist.chpl:163</text>
+</g>
+<!-- 1940492027&#45;&gt;&#45;1223359704 -->
+<g id="edge106" class="edge"><title>1940492027&#45;&gt;&#45;1223359704</title>
+<path fill="none" stroke="black" d="M1833.3,-1040.49C1833.6,-1028.74 1834.07,-1010.21 1834.44,-995.696"/>
+<polygon fill="black" stroke="black" points="1837.95,-995.643 1834.7,-985.556 1830.95,-995.463 1837.95,-995.643"/>
+<text text-anchor="middle" x="1861.24" y="-1008.8" font-family="Times,serif" font-size="14.00">[1&#45;28,30]</text>
+</g>
+<!-- &#45;706931770 -->
+<g id="node55" class="node"><title>&#45;706931770</title>
+<polygon fill="#aaaaaa" stroke="black" points="1619.34,-880.6 1619.34,-905.4 2050.66,-905.4 2050.66,-880.6 1619.34,-880.6"/>
+<text text-anchor="middle" x="1835" y="-888.8" font-family="Times,serif" font-size="14.00" fill="#000000">on_fn_chpl64$$CFE_id_f9ee5c55_9b0c5384@SparseBlockDist.chpl:164</text>
+</g>
+<!-- &#45;1223359704&#45;&gt;&#45;706931770 -->
+<g id="edge108" class="edge"><title>&#45;1223359704&#45;&gt;&#45;706931770</title>
+<path fill="none" stroke="black" d="M1835,-960.491C1835,-948.739 1835,-930.212 1835,-915.696"/>
+<polygon fill="black" stroke="black" points="1838.5,-915.556 1835,-905.556 1831.5,-915.556 1838.5,-915.556"/>
+<text text-anchor="middle" x="1861.24" y="-928.8" font-family="Times,serif" font-size="14.00">[1&#45;28,30]</text>
+</g>
+<!-- 639807108 -->
+<g id="node56" class="node"><title>639807108</title>
+<polygon fill="#aaaaaa" stroke="black" points="1657.12,-800.6 1657.12,-825.4 2044.88,-825.4 2044.88,-800.6 1657.12,-800.6"/>
+<text text-anchor="middle" x="1851" y="-808.8" font-family="Times,serif" font-size="14.00" fill="#000000">bulkAdd$$CFE_id_f9ee5c55_9b0c5384@ChapelArray.chpl:1336</text>
+</g>
+<!-- &#45;706931770&#45;&gt;639807108 -->
+<g id="edge110" class="edge"><title>&#45;706931770&#45;&gt;639807108</title>
+<path fill="none" stroke="black" d="M1837.36,-880.491C1839.77,-868.739 1843.57,-850.212 1846.55,-835.696"/>
+<polygon fill="black" stroke="black" points="1850.05,-836.056 1848.63,-825.556 1843.19,-834.649 1850.05,-836.056"/>
+<text text-anchor="middle" x="1871.24" y="-848.8" font-family="Times,serif" font-size="14.00">[1&#45;28,30]</text>
+</g>
+<!-- &#45;927059089 -->
+<g id="node57" class="node"><title>&#45;927059089</title>
+<polygon fill="#aaaaaa" stroke="black" points="1642.27,-720.6 1642.27,-745.4 2083.73,-745.4 2083.73,-720.6 1642.27,-720.6"/>
+<text text-anchor="middle" x="1863" y="-728.8" font-family="Times,serif" font-size="14.00" fill="#000000">dsiBulkAdd2$$CFE_id_f9ee5c55_9b0c5384@ChapelDistribution.chpl:410</text>
+</g>
+<!-- 639807108&#45;&gt;&#45;927059089 -->
+<g id="edge112" class="edge"><title>639807108&#45;&gt;&#45;927059089</title>
+<path fill="none" stroke="black" d="M1852.77,-800.491C1854.58,-788.739 1857.43,-770.212 1859.66,-755.696"/>
+<polygon fill="black" stroke="black" points="1863.16,-755.972 1861.22,-745.556 1856.24,-754.908 1863.16,-755.972"/>
+<text text-anchor="middle" x="1885.24" y="-768.8" font-family="Times,serif" font-size="14.00">[1&#45;28,30]</text>
+</g>
+<!-- 1840016963 -->
+<g id="node58" class="node"><title>1840016963</title>
+<polygon fill="#aaaaaa" stroke="black" points="1644.6,-640.6 1644.6,-665.4 2081.4,-665.4 2081.4,-640.6 1644.6,-640.6"/>
+<text text-anchor="middle" x="1863" y="-648.8" font-family="Times,serif" font-size="14.00" fill="#000000">bulkAdd_help_chpl$$CFE_id_f9ee5c55_9b0c5384@LayoutCSR.chpl:300</text>
+</g>
+<!-- &#45;927059089&#45;&gt;1840016963 -->
+<g id="edge114" class="edge"><title>&#45;927059089&#45;&gt;1840016963</title>
+<path fill="none" stroke="black" d="M1863,-720.491C1863,-708.739 1863,-690.212 1863,-675.696"/>
+<polygon fill="black" stroke="black" points="1866.5,-675.556 1863,-665.556 1859.5,-675.556 1866.5,-675.556"/>
+<text text-anchor="middle" x="1889.24" y="-688.8" font-family="Times,serif" font-size="14.00">[1&#45;28,30]</text>
+</g>
+<!-- &#45;1651790862 -->
+<g id="node59" class="node"><title>&#45;1651790862</title>
+<polygon fill="#9c71c7" stroke="black" points="1669.68,-560.6 1669.68,-585.4 1846.32,-585.4 1846.32,-560.6 1669.68,-560.6"/>
+<text text-anchor="middle" x="1758" y="-568.8" font-family="Times,serif" font-size="14.00" fill="#000000">__cray_EXP2@0x2058d6cd</text>
+</g>
+<!-- 1840016963&#45;&gt;&#45;1651790862 -->
+<g id="edge116" class="edge"><title>1840016963&#45;&gt;&#45;1651790862</title>
+<path fill="none" stroke="black" d="M1845.83,-640.558C1837.72,-635.061 1827.96,-628.31 1819.36,-622 1806.2,-612.342 1791.81,-601.121 1780.27,-591.957"/>
+<polygon fill="black" stroke="black" points="1782.43,-589.205 1772.43,-585.7 1778.07,-594.675 1782.43,-589.205"/>
+<text text-anchor="middle" x="1864.32" y="-608.8" font-family="Times,serif" font-size="14.00">[1&#45;10,12&#45;28,30]</text>
+</g>
+<!-- &#45;38075676 -->
+<g id="node61" class="node"><title>&#45;38075676</title>
+<polygon fill="#8e38e3" stroke="black" points="1877.18,-560.6 1877.18,-585.4 2074.82,-585.4 2074.82,-560.6 1877.18,-560.6"/>
+<text text-anchor="middle" x="1976" y="-568.8" font-family="Times,serif" font-size="14.00" fill="#ffffff">__cray_EXP2_03@0x2058d6cd</text>
+</g>
+<!-- 1840016963&#45;&gt;&#45;38075676 -->
+<g id="edge120" class="edge"><title>1840016963&#45;&gt;&#45;38075676</title>
+<path fill="none" stroke="black" d="M1880.6,-640.425C1888.94,-634.899 1899.02,-628.162 1908,-622 1922.52,-612.036 1938.63,-600.681 1951.52,-591.516"/>
+<polygon fill="black" stroke="black" points="1953.76,-594.219 1959.87,-585.566 1949.7,-588.518 1953.76,-594.219"/>
+<text text-anchor="middle" x="1944.66" y="-608.8" font-family="Times,serif" font-size="14.00">[11]</text>
+</g>
+<!-- &#45;874276446 -->
+<g id="node60" class="node"><title>&#45;874276446</title>
+<polygon fill="#9c71c7" stroke="black" points="1644.01,-480.6 1644.01,-505.4 1857.99,-505.4 1857.99,-480.6 1644.01,-480.6"/>
+<text text-anchor="middle" x="1751" y="-488.8" font-family="Times,serif" font-size="14.00" fill="#000000">__cray_EXP2_Z_03@0x2058d70b</text>
+</g>
+<!-- &#45;1651790862&#45;&gt;&#45;874276446 -->
+<g id="edge118" class="edge"><title>&#45;1651790862&#45;&gt;&#45;874276446</title>
+<path fill="none" stroke="black" d="M1756.97,-560.491C1755.91,-548.739 1754.25,-530.212 1752.95,-515.696"/>
+<polygon fill="black" stroke="black" points="1756.42,-515.203 1752.04,-505.556 1749.45,-515.829 1756.42,-515.203"/>
+<text text-anchor="middle" x="1800.32" y="-528.8" font-family="Times,serif" font-size="14.00">[1&#45;10,12&#45;28,30]</text>
+</g>
+<!-- &#45;1131396048 -->
+<g id="node62" class="node"><title>&#45;1131396048</title>
+<polygon fill="#8e38e3" stroke="black" points="1876.01,-480.6 1876.01,-505.4 2089.99,-505.4 2089.99,-480.6 1876.01,-480.6"/>
+<text text-anchor="middle" x="1983" y="-488.8" font-family="Times,serif" font-size="14.00" fill="#ffffff">__cray_EXP2_Z_03@0x2058d70b</text>
+</g>
+<!-- &#45;38075676&#45;&gt;&#45;1131396048 -->
+<g id="edge122" class="edge"><title>&#45;38075676&#45;&gt;&#45;1131396048</title>
+<path fill="none" stroke="black" d="M1977.03,-560.491C1978.09,-548.739 1979.75,-530.212 1981.05,-515.696"/>
+<polygon fill="black" stroke="black" points="1984.55,-515.829 1981.96,-505.556 1977.58,-515.203 1984.55,-515.829"/>
+<text text-anchor="middle" x="1992.66" y="-528.8" font-family="Times,serif" font-size="14.00">[11]</text>
+</g>
+</g>
+</svg>