Sketch of generating zxy streams from S3 sources

test/travis-resources.template → cloudformation/travis.template

@@ -42,6 +42,29 @@
                             ]
                         }
                     },
+                    {
+                        "PolicyName": "testList",
+                        "PolicyDocument": {
+                            "Statement": [
+                                {
+                                    "Action": [
+                                        "s3:ListBucket"
+                                    ],
+                                    "Effect": "Allow",
+                                    "Resource": [
+                                        "arn:aws:s3:::mapbox"
+                                    ],
+                                    "Condition": {
+                                        "StringLike": {
+                                            "s3:prefix": [
+                                                "tilelive-s3/*"
+                                            ]
+                                        }
+                                    }
+                                }
+                            ]
+                        }
+                    },
                     {
                         "PolicyName": "test",
                         "PolicyDocument": {

lib/index.js

@@ -15,6 +15,7 @@ var AgentKeepAlive = require('agentkeepalive');
 var s3urls = require('s3urls');
 var utils = require('util');
 var Emitter = require('events').EventEmitter;
+var ZXYStream = require('./zxystream');
 
 module.exports = S3;
 
@@ -601,3 +602,11 @@ S3.prototype.getIndexableDocs = function(pointer, callback) {
         callback(null, docs, pointer);
     });
 };
+
+// Create a readable ZXY list stream.
+S3.prototype.createZXYStream = function() {
+    if (!this.data) return callback(new Error('Tilesource not loaded'));
+    if (!this.data.tiles) return callback(new Error('No "tiles" key'));
+    return new ZXYStream(this.data.tiles[0]);
+};
+

lib/zxystream.js

@@ -0,0 +1,57 @@
+var util = require('util');
+var split = require('split');
+var s3scan = require('s3scan');
+var s3urls = require('s3urls');
+var StreamConcat = require('stream-concat');
+
+module.exports = ZXYStream;
+module.exports.config = config;
+
+function ZXYStream(tilesUrl) {
+    var conf = config(tilesUrl);
+    var current = 0;
+    return new StreamConcat(function() {
+        if (!conf.toList[current]) return null;
+        return s3scan.List(conf.toList[current++], {})
+    }).pipe(split(function(line) {
+        var matches = line.match(conf.tokenPattern);
+        if (!matches || matches.length < 4) return undefined;
+        return matches[conf.tokenMap[0]] + '/' +
+            matches[conf.tokenMap[1]] + '/' +
+            matches[conf.tokenMap[2]] + '\n';
+    }));
+}
+
+function config(tilesUrl) {
+    var conf = {};
+    conf.toList = [];
+    if ((/{prefix}/).test(tilesUrl)) {
+        for (var i = 0; i < 256; i++) {
+            var prefix = i < 16 ? '0' + i.toString(16) : i.toString(16);
+            conf.toList.push(s3urls.convert(tilesUrl, 's3').split(/{(z|x|y)}/)[0].replace(/{prefix}/g,prefix));
+        }
+    } else {
+        conf.toList.push(s3urls.convert(tilesUrl, 's3').split(/{(z|x|y)}/)[0]);
+    }
+
+    var matches = s3urls.fromUrl(tilesUrl).Key.match(/{(z|x|y)}/g);
+    if (matches.length !== 3) throw new Error('Could not find {z}, {x} and {y} tokens in url: ' + tilesUrl);
+
+    conf.tokenMap = matches.reduce(function(memo, token, i) {
+        if (token === '{z}') {
+            memo[i] = 1;
+        } else if (token === '{x}') {
+            memo[i] = 2;
+        } else if (token === '{y}') {
+            memo[i] = 3;
+        }
+        return memo;
+    }, []);
+
+    conf.tokenPattern = new RegExp(s3urls.fromUrl(tilesUrl).Key
+        .replace(/{prefix}/g, '[0-f]{2}')
+        .replace(/{(z|x|y)}/g, '([0-9]+)'));
+
+    return conf;
+}
+

package.json

@@ -6,7 +6,10 @@
     "aws-sdk": "^2.1.44",
     "nan": "^1.8.4",
     "node-pre-gyp": "~0.6.7",
+    "split": "~1.0.0",
+    "s3scan": "0.1.0",
     "s3urls": "^1.3.0",
+    "stream-concat": "0.1.0",
     "tiletype": "0.1.x"
   },
   "bundledDependencies": [

test/s3.test.js

@@ -629,3 +629,12 @@ tape('source load error should fail gracefully', function(assert) {
         });
     });
 })();
+
+tape('createZXYStream', function(assert) {
+    new S3(url.parse('s3://mapbox/tilelive-s3/test/{z}/{x}/{y}.png'), function(err, source) {
+        assert.ifError(err, 'success');
+        var zxyStream = source.createZXYStream();
+        assert.equal(zxyStream.readable, true, 'returns readable stream');
+        assert.end();
+    });
+});