Improves Chunker to split content based on a given terminator.

diff --git a/lib/selectors/tokenizer.js b/lib/selectors/tokenizer.js
index ed074c9..3e960f9 100644 (file)
--- a/lib/selectors/tokenizer.js
+++ b/lib/selectors/tokenizer.js
@@ -1,7 +1,7 @@
 var Chunker = require('../utils/chunker');
 
 module.exports = function Tokenizer(data, minifyContext) {
-  var chunker = new Chunker(data, 128);
+  var chunker = new Chunker(data, '}', 128);
   var chunk = chunker.next();
   var flatBlock = /(^@(font\-face|page|\-ms\-viewport|\-o\-viewport|viewport)|\\@.+?)/;
 

diff --git a/lib/utils/chunker.js b/lib/utils/chunker.js
index 73dc70e..860a425 100644 (file)
--- a/lib/utils/chunker.js
+++ b/lib/utils/chunker.js
@@ -1,4 +1,4 @@
-module.exports = function Chunker(data, chunkSize) {
+module.exports = function Chunker(data, breakString, chunkSize) {
 // Divides `data` into chunks of `chunkSize` for faster processing
   var chunks = [];
   for (var cursor = 0, dataSize = data.length; cursor < dataSize;) {
@@ -6,13 +6,13 @@ module.exports = function Chunker(data, chunkSize) {
       dataSize - 1 :
       cursor + chunkSize;
 
-    if (data[nextCursor] != '}')
-      nextCursor = data.indexOf('}', nextCursor);
+    if (data[nextCursor] != breakString)
+      nextCursor = data.indexOf(breakString, nextCursor);
     if (nextCursor == -1)
       nextCursor = data.length - 1;
 
-    chunks.push(data.substring(cursor, nextCursor + 1));
-    cursor = nextCursor + 1;
+    chunks.push(data.substring(cursor, nextCursor + breakString.length));
+    cursor = nextCursor + breakString.length;
   }
 
   return {

diff --git a/test/utils/chunker-test.js b/test/utils/chunker-test.js
index f56a38d..c337e0a 100644 (file)
--- a/test/utils/chunker-test.js
+++ b/test/utils/chunker-test.js
@@ -5,7 +5,7 @@ var Chunker = require('../../lib/utils/chunker');
 vows.describe(Chunker)
   .addBatch({
     'empty string': {
-      topic: new Chunker('', 128),
+      topic: new Chunker('', '', 128),
       'is empty': function (chunker) {
         assert.isTrue(chunker.isEmpty());
       },
@@ -14,7 +14,7 @@ vows.describe(Chunker)
       }
     },
     'css': {
-      topic: new Chunker('a{color:red}p{}', 3),
+      topic: new Chunker('a{color:red}p{}', '}', 3),
       'is not empty': function (chunker) {
         assert.isFalse(chunker.isEmpty());
       },
@@ -24,6 +24,18 @@ vows.describe(Chunker)
       'breaks at second brace': function (chunker) {
         assert.equal('p{}', chunker.next());
       }
+    },
+    'comments': {
+      topic: new Chunker('/* one */ /* two */', '*/', 3),
+      'is not empty': function (chunker) {
+        assert.isFalse(chunker.isEmpty());
+      },
+      'breaks at first brace': function (chunker) {
+        assert.equal('/* one */', chunker.next());
+      },
+      'breaks at second brace': function (chunker) {
+        assert.equal(' /* two */', chunker.next());
+      }
     }
   })
   .export(module);