add below to scattermapbox + handle below:'traces' in layout layers

We now handle below logic (mostly) at the subplot level in
  subplot.fillBelowLookup() and subplot.belowLookup.
  This allows us to handle many edges cases in a cleaner way.

In brief,
  + we collect set trace/layout-layer below value,
    OR find their "smart" default value
  + each trace/layout-layer stash a below state value
  + if new below doesn't match old before, we must remove/add layer
  + if many traces/layout have same below value, we place choroplethmapbox,
    then densitymapbox, scattermapbox and finally layout layers
    in that order.
  + on update, if many traces/layout-layer have same below,
    we must in general remove/add all those layers
    to have the correct ordering

Additional notes:
  - we no longer need to handle below in convert.js routines
  - getBelow is now a trace _module.getBelow method, so that it
    can get called before mapbox-trace object creation
  - new subplot.getMapLayer method to DRY things up

Author: etpinard

Diff for: src/plots/mapbox/layers.js

@@ -17,6 +17,7 @@ function MapboxLayer(subplot, index) {
     this.map = subplot.map;
 
     this.uid = subplot.uid + '-' + index;
+    this.index = index;
 
     this.idSource = 'source-' + this.uid;
     this.idLayer = constants.layoutLayerPrefix + this.uid;
@@ -66,7 +67,7 @@ proto.needsNewSource = function(opts) {
 proto.needsNewLayer = function(opts) {
     return (
         this.layerType !== opts.type ||
-        this.below !== opts.below
+        this.below !== this.subplot.belowLookup['layout-' + this.index]
     );
 };
 
@@ -89,8 +90,27 @@ proto.updateLayer = function(opts) {
     var map = this.map;
     var convertedOpts = convertOpts(opts);
 
+    var below = this.subplot.belowLookup['layout-' + this.index];
+    var _below;
+
+    if(below === 'traces') {
+        var mapLayers = this.subplot.getMapLayers();
+
+        // find id of first plotly trace layer
+        for(var i = 0; i < mapLayers.length; i++) {
+            var layerId = mapLayers[i].id;
+            if(typeof layerId === 'string' &&
+                layerId.indexOf(constants.traceLayerPrefix) === 0
+            ) {
+                _below = layerId;
+                break;
+            }
+        }
+    } else {
+        _below = below;
+    }
+
     this.removeLayer();
-    this.layerType = opts.type;
 
     if(isVisible(opts)) {
         map.addLayer({
@@ -102,8 +122,11 @@ proto.updateLayer = function(opts) {
             maxzoom: opts.maxzoom,
             layout: convertedOpts.layout,
             paint: convertedOpts.paint
-        }, opts.below);
+        }, _below);
     }
+
+    this.layerType = opts.type;
+    this.below = below;
 };
 
 proto.updateStyle = function(opts) {

Diff for: src/plots/mapbox/layout_attributes.js

@@ -158,7 +158,6 @@ var attrs = module.exports = overrideAll({
         // attributes shared between all types
         below: {
             valType: 'string',
-            dflt: '',
             role: 'info',
             description: [
                 'Determines if the layer will be inserted',

Diff for: src/plots/mapbox/mapbox.js

@@ -48,6 +48,7 @@ function Mapbox(gd, id) {
     this.styleObj = null;
     this.traceHash = {};
     this.layerList = [];
+    this.belowLookup = {};
 }
 
 var proto = Mapbox.prototype;
@@ -126,6 +127,7 @@ proto.createMap = function(calcData, fullLayout, resolve, reject) {
     promises = promises.concat(self.fetchMapData(calcData, fullLayout));
 
     Promise.all(promises).then(function() {
+        self.fillBelowLookup(calcData, fullLayout);
         self.updateData(calcData);
         self.updateLayout(fullLayout);
         self.resolveOnRender(resolve);
@@ -191,12 +193,94 @@ proto.updateMap = function(calcData, fullLayout, resolve, reject) {
     promises = promises.concat(self.fetchMapData(calcData, fullLayout));
 
     Promise.all(promises).then(function() {
+        self.fillBelowLookup(calcData, fullLayout);
         self.updateData(calcData);
         self.updateLayout(fullLayout);
         self.resolveOnRender(resolve);
     }).catch(reject);
 };
 
+proto.fillBelowLookup = function(calcData, fullLayout) {
+    var opts = fullLayout[this.id];
+    var layers = opts.layers;
+    var i, val;
+
+    var belowLookup = this.belowLookup = {};
+    var hasTraceAtTop = false;
+
+    for(i = 0; i < calcData.length; i++) {
+        var trace = calcData[i][0].trace;
+        var _module = trace._module;
+
+        if(typeof trace.below === 'string') {
+            val = trace.below;
+        } else if(_module.getBelow) {
+            // 'smart' default that depend the map's base layers
+            val = _module.getBelow(trace, this);
+        }
+
+        if(val === '') {
+            hasTraceAtTop = true;
+        }
+
+        belowLookup['trace-' + trace.uid] = val || '';
+    }
+
+    for(i = 0; i < layers.length; i++) {
+        var item = layers[i];
+
+        if(typeof item.below === 'string') {
+            val = item.below;
+        } else if(hasTraceAtTop) {
+            // if one or more trace(s) set `below:''` and
+            // layers[i].below is unset,
+            // place layer below traces
+            val = 'traces';
+        } else {
+            val = '';
+        }
+
+        belowLookup['layout-' + i] = val;
+    }
+
+    // N.B. If multiple layers have the 'below' value,
+    // we must clear the stashed 'below' field in order
+    // to make `traceHash[k].update()` and `layerList[i].update()`
+    // remove/add the all those layers to have preserve
+    // the correct layer ordering
+    var val2list = {};
+    var k, id;
+
+    for(k in belowLookup) {
+        val = belowLookup[k];
+        if(val2list[val]) {
+            val2list[val].push(k);
+        } else {
+            val2list[val] = [k];
+        }
+    }
+
+    for(val in val2list) {
+        var list = val2list[val];
+        if(list.length > 1) {
+            for(i = 0; i < list.length; i++) {
+                k = list[i];
+                if(k.indexOf('trace-') === 0) {
+                    id = k.split('trace-')[1];
+                    if(this.traceHash[id]) {
+                        this.traceHash[id].below = null;
+                    }
+                } else if(k.indexOf('layout-') === 0) {
+                    id = k.split('layout-')[1];
+                    if(this.layerList[id]) {
+                        this.layerList[id].below = null;
+                    }
+                }
+            }
+        }
+    }
+};
+
 var traceType2orderIndex = {
     choroplethmapbox: 0,
     densitymapbox: 1,
@@ -207,6 +291,10 @@ proto.updateData = function(calcData) {
     var traceHash = this.traceHash;
     var traceObj, trace, i, j;
 
+    // Need to sort here by trace type here,
+    // in case traces with different `type` have the same
+    // below value, but sorting we ensure that
+    // e.g. choroplethmapbox traces will be below scattermapbox traces
     var calcDataSorted = calcData.slice().sort(function(a, b) {
         return (
             traceType2orderIndex[a[0].trace.type] -
@@ -598,6 +686,10 @@ proto.setOptions = function(id, methodName, opts) {
     }
 };
 
+proto.getMapLayers = function() {
+    return this.map.getStyle().layers;
+};
+
 // convenience method to project a [lon, lat] array to pixel coords
 proto.project = function(v) {
     return this.map.project(new mapboxgl.LngLat(v[0], v[1]));

Diff for: src/traces/choroplethmapbox/convert.js

@@ -176,7 +176,6 @@ function convert(calcTrace) {
     line.layout.visibility = 'visible';
 
     opts.geojson = {type: 'FeatureCollection', features: featuresOut};
-    opts.below = trace.below;
 
     convertOnSelect(calcTrace);
 

Diff for: src/traces/choroplethmapbox/index.js

@@ -25,6 +25,30 @@ module.exports = {
         }
     },
 
+    getBelow: function(trace, subplot) {
+        var mapLayers = subplot.getMapLayers();
+
+        // find layer just above top-most "water" layer
+        // that is not a plotly layer
+        for(var i = mapLayers.length - 2; i >= 0; i--) {
+            var layerId = mapLayers[i].id;
+
+            if(typeof layerId === 'string' &&
+                layerId.indexOf('water') === 0
+             ) {
+                for(var j = i + 1; j < mapLayers.length; j++) {
+                    layerId = mapLayers[j].id;
+
+                    if(typeof layerId === 'string' &&
+                        layerId.indexOf('plotly-') === -1
+                    ) {
+                        return layerId;
+                    }
+                }
+            }
+        }
+    },
+
     moduleType: 'trace',
     name: 'choroplethmapbox',
     basePlotModule: require('../../plots/mapbox'),

Diff for: src/traces/choroplethmapbox/plot.js

@@ -42,14 +42,16 @@ proto.updateOnSelect = function(calcTrace) {
 proto._update = function(optsAll) {
     var subplot = this.subplot;
     var layerList = this.layerList;
+    var below = subplot.belowLookup['trace-' + this.uid];
 
     subplot.map
         .getSource(this.sourceId)
         .setData(optsAll.geojson);
 
-    if(optsAll.below !== this.below) {
+    if(below !== this.below) {
         this._removeLayers();
-        this._addLayers(optsAll);
+        this._addLayers(optsAll, below);
+        this.below = below;
     }
 
     for(var i = 0; i < layerList.length; i++) {
@@ -66,11 +68,10 @@ proto._update = function(optsAll) {
     }
 };
 
-proto._addLayers = function(optsAll) {
+proto._addLayers = function(optsAll, below) {
     var subplot = this.subplot;
     var layerList = this.layerList;
     var sourceId = this.sourceId;
-    var below = this.getBelow(optsAll);
 
     for(var i = 0; i < layerList.length; i++) {
         var item = layerList[i];
@@ -85,8 +86,6 @@ proto._addLayers = function(optsAll) {
             paint: opts.paint
         }, below);
     }
-
-    this.below = below;
 };
 
 proto._removeLayers = function() {
@@ -104,47 +103,19 @@ proto.dispose = function() {
     map.removeSource(this.sourceId);
 };
 
-proto.getBelow = function(optsAll) {
-    if(optsAll.below !== undefined) {
-        return optsAll.below;
-    }
-
-    var mapLayers = this.subplot.map.getStyle().layers;
-    var out = '';
-
-    // find layer just above top-most "water" layer
-    for(var i = 0; i < mapLayers.length; i++) {
-        var layerId = mapLayers[i].id;
-
-        if(typeof layerId === 'string') {
-            var isWaterLayer = layerId.indexOf('water') === 0;
-
-            if(out && !isWaterLayer) {
-                out = layerId;
-                break;
-            }
-            if(isWaterLayer) {
-                out = layerId;
-            }
-        }
-    }
-
-    return out;
-};
-
 module.exports = function createChoroplethMapbox(subplot, calcTrace) {
     var trace = calcTrace[0].trace;
     var choroplethMapbox = new ChoroplethMapbox(subplot, trace.uid);
     var sourceId = choroplethMapbox.sourceId;
-
     var optsAll = convert(calcTrace);
+    var below = choroplethMapbox.below = subplot.belowLookup['trace-' + trace.uid];
 
     subplot.map.addSource(sourceId, {
         type: 'geojson',
         data: optsAll.geojson
     });
 
-    choroplethMapbox._addLayers(optsAll);
+    choroplethMapbox._addLayers(optsAll, below);
 
     // link ref for quick update during selections
     calcTrace[0].trace._glTrace = choroplethMapbox;

Diff for: src/traces/densitymapbox/convert.js

@@ -109,7 +109,6 @@ module.exports = function convert(calcTrace) {
 
     opts.geojson = {type: 'FeatureCollection', features: features};
     opts.heatmap.layout.visibility = 'visible';
-    opts.below = trace.below;
 
     return opts;
 };

Diff for: src/traces/densitymapbox/index.js

@@ -17,6 +17,22 @@ module.exports = {
     hoverPoints: require('./hover'),
     eventData: require('./event_data'),
 
+    getBelow: function(trace, subplot) {
+        var mapLayers = subplot.getMapLayers();
+
+        // find first layer with `type: 'symbol'`,
+        // that is not a plotly layer
+        for(var i = 0; i < mapLayers.length; i++) {
+            var layer = mapLayers[i];
+            var layerId = layer.id;
+            if(layer.type === 'symbol' &&
+                typeof layerId === 'string' && layerId.indexOf('plotly-') === -1
+            ) {
+                return layerId;
+            }
+        }
+    },
+
     moduleType: 'trace',
     name: 'densitymapbox',
     basePlotModule: require('../../plots/mapbox'),