Replaces Chunker with uber-powerful split.

These two were doing basically the same thing, while the latter
being smarter about a context.

diff --git a/lib/tokenizer/chunker.js b/lib/tokenizer/chunker.js
deleted file mode 100644 (file)
index 438b8ba..0000000
--- a/lib/tokenizer/chunker.js
+++ /dev/null
@@ -1,28 +0,0 @@
-// Divides `data` into chunks of `chunkSize` for faster processing
-function Chunker(data, breakString, chunkSize) {
-  this.chunks = [];
-
-  for (var cursor = 0, dataSize = data.length; cursor < dataSize;) {
-    var nextCursor = cursor + chunkSize > dataSize ?
-      dataSize - 1 :
-      cursor + chunkSize;
-
-    if (data[nextCursor] != breakString)
-      nextCursor = data.indexOf(breakString, nextCursor);
-    if (nextCursor == -1)
-      nextCursor = data.length - 1;
-
-    this.chunks.push(data.substring(cursor, nextCursor + breakString.length));
-    cursor = nextCursor + breakString.length;
-  }
-}
-
-Chunker.prototype.isEmpty = function () {
-  return this.chunks.length === 0;
-};
-
-Chunker.prototype.next = function () {
-  return this.chunks.shift();
-};
-
-module.exports = Chunker;

diff --git a/lib/tokenizer/tokenize.js b/lib/tokenizer/tokenize.js
index 6fd5f47..3061dea 100644 (file)
--- a/lib/tokenizer/tokenize.js
+++ b/lib/tokenizer/tokenize.js
@@ -1,4 +1,3 @@
-var Chunker = require('./chunker');
 var extractProperties = require('./extract-properties');
 var extractSelectors = require('./extract-selectors');
 var track = require('../source-maps/track');
@@ -8,13 +7,13 @@ var path = require('path');
 var flatBlock = /(@(font\-face|page|\-ms\-viewport|\-o\-viewport|viewport|counter\-style)|\\@.+?)/;
 
 function tokenize(data, outerContext) {
-  var chunker = new Chunker(normalize(data), '}', 128);
-  if (chunker.isEmpty())
+  var chunks = split(normalize(data), '}', true, '{', '}');
+  if (chunks.length === 0)
     return [];
 
   var context = {
-    chunk: chunker.next(),
-    chunker: chunker,
+    chunk: chunks.shift(),
+    chunks: chunks,
     column: 0,
     cursor: 0,
     line: 1,
@@ -58,10 +57,10 @@ function whatsNext(context) {
   var closest;
 
   if (chunk.length == context.cursor) {
-    if (context.chunker.isEmpty())
+    if (context.chunks.length === 0)
       return null;
 
-    context.chunk = chunk = context.chunker.next();
+    context.chunk = chunk = context.chunks.shift();
     context.cursor = 0;
   }
 

diff --git a/test/module-test.js b/test/module-test.js
index 6dd21be..c6ac3f2 100644 (file)
--- a/test/module-test.js
+++ b/test/module-test.js
@@ -123,7 +123,7 @@ vows.describe('module tests').addBatch({
     },
     'should raise one warning': function (error, minified) {
       assert.lengthOf(minified.warnings, 1);
-      assert.equal(minified.warnings[0], 'Unexpected \'}\' in \'a{display:block}}\'. Ignoring.');
+      assert.equal(minified.warnings[0], 'Unexpected \'}\' in \'}\'. Ignoring.');
     }
   },
   'warnings on missing closing brace': {

diff --git a/test/tokenizer/chunker-test.js b/test/tokenizer/chunker-test.js
deleted file mode 100644 (file)
index 3c4861d..0000000
--- a/test/tokenizer/chunker-test.js
+++ /dev/null
@@ -1,47 +0,0 @@
-var vows = require('vows');
-var assert = require('assert');
-var Chunker = require('../../lib/tokenizer/chunker');
-
-vows.describe(Chunker)
-  .addBatch({
-    'empty string': {
-      'topic': function () {
-        return new Chunker('', '', 128);
-      },
-      'is empty': function (chunker) {
-        assert.isTrue(chunker.isEmpty());
-      },
-      'has no next': function (chunker) {
-        assert.isUndefined(chunker.next());
-      }
-    },
-    'css': {
-      'topic': function () {
-        return new Chunker('a{color:red}p{}', '}', 3);
-      },
-      'is not empty': function (chunker) {
-        assert.isFalse(chunker.isEmpty());
-      },
-      'breaks at first brace': function (chunker) {
-        assert.equal(chunker.next(), 'a{color:red}');
-      },
-      'breaks at second brace': function (chunker) {
-        assert.equal(chunker.next(), 'p{}');
-      }
-    },
-    'comments': {
-      'topic': function () {
-        return new Chunker('/* one */ /* two */', '*/', 3);
-      },
-      'is not empty': function (chunker) {
-        assert.isFalse(chunker.isEmpty());
-      },
-      'breaks at first brace': function (chunker) {
-        assert.equal(chunker.next(), '/* one */');
-      },
-      'breaks at second brace': function (chunker) {
-        assert.equal(chunker.next(), ' /* two */');
-      }
-    }
-  })
-  .export(module);