Rewrites property tokenizer to use string traversing.

diff --git a/lib/selectors/tokenizer.js b/lib/selectors/tokenizer.js
index 8717d31..0bd21d3 100644 (file)
--- a/lib/selectors/tokenizer.js
+++ b/lib/selectors/tokenizer.js
@@ -4,19 +4,15 @@ var Splitter = require('../utils/splitter');
 var flatBlock = /(^@(font\-face|page|\-ms\-viewport|\-o\-viewport|viewport|counter\-style)|\\@.+?)/;
 var WHITESPACE = /\s/g;
 var MULTI_WHITESPACE = /\s{2,}/g;
-var WHITESPACE_COLON = / ?: ?/g;
-var WHITESPACE_SEMICOLON = / ?; ?/g;
 var WHITESPACE_COMMA = / ?, ?/g;
-var WHITESPACE_PREFIX = /([\(,]) /g;
-var WHITESPACE_SUFFIX = / ([\),])/g;
-var MULTI_SEMICOLON = /;{2,}/g;
-var TRAILING_SEMICOLON = /;$/;
 
 function Tokenizer(minifyContext) {
   this.minifyContext = minifyContext;
 }
 
 Tokenizer.prototype.toTokens = function (data) {
+  data = data.replace(/\r\n/g, '\n');
+
   var chunker = new Chunker(data, '}', 128);
   if (chunker.isEmpty())
     return [];
@@ -32,21 +28,55 @@ Tokenizer.prototype.toTokens = function (data) {
   return tokenize(context);
 };
 
+function valueMapper(property) { return { value: property }; }
+
 function extractProperties(string) {
-  var extracted = string
-    .replace(WHITESPACE, ' ')
-    .replace(MULTI_WHITESPACE, ' ')
-    .replace(WHITESPACE_COLON, ':')
-    .replace(WHITESPACE_PREFIX, '$1')
-    .replace(WHITESPACE_SUFFIX, '$1')
-    .replace(WHITESPACE_SEMICOLON, ';')
-    .replace(MULTI_SEMICOLON, ';')
-    .trim()
-    .replace(TRAILING_SEMICOLON, '');
-
-  return extracted.length > 0 ?
-    extracted.split(';').map(function (property) { return { value: property }; }) :
-    [];
+  var properties = [];
+  var buffer = [];
+  var isWhitespace;
+  var wasWhitespace;
+  var isSpecial;
+  var wasSpecial;
+  var current;
+  var wasCloseParenthesis;
+
+  for (var i = 0, l = string.length; i < l; i++) {
+    current = string[i];
+
+    if (current === ';') {
+      if (wasWhitespace && buffer[buffer.length - 1] === ' ')
+        buffer.pop();
+      if (buffer.length > 0)
+        properties.push({ value: buffer.join('') });
+      buffer = [];
+    } else {
+      isWhitespace = current === ' ' || current === '\t' || current === '\n';
+      isSpecial = current === ':' || current === '[' || current === ']' || current === ',' || current === '(' || current === ')';
+
+      if (wasWhitespace && isSpecial) {
+        buffer.pop();
+        buffer.push(current);
+      } else if (isWhitespace && wasSpecial && !wasCloseParenthesis) {
+      } else if (isWhitespace && !wasWhitespace && buffer.length > 0) {
+        buffer.push(' ');
+      } else if (isWhitespace && buffer.length === 0) {
+      } else if (isWhitespace && wasWhitespace) {
+      } else {
+        buffer.push(isWhitespace ? ' ' : current);
+      }
+    }
+
+    wasSpecial = isSpecial;
+    wasWhitespace = isWhitespace;
+    wasCloseParenthesis = current === ')';
+  }
+
+  if (wasWhitespace && buffer[buffer.length - 1] === ' ')
+    buffer.pop();
+  if (buffer.length > 0)
+    properties.push({ value: buffer.join('') });
+
+  return properties;
 }
 
 function extractSelectors(string) {
@@ -56,9 +86,7 @@ function extractSelectors(string) {
     .replace(WHITESPACE_COMMA, ',')
     .trim();
 
-  return new Splitter(',').split(extracted).map(function (selector) {
-    return { value: selector };
-  });
+  return new Splitter(',').split(extracted).map(valueMapper);
 }
 
 function extractBlock(string) {

diff --git a/test/integration-test.js b/test/integration-test.js
index 5e06508..0f7d6fa 100644 (file)
--- a/test/integration-test.js
+++ b/test/integration-test.js
@@ -96,10 +96,6 @@ vows.describe('integration tests').addBatch({
       'div \na\r\n, p { width:500px }',
       'div a,p{width:500px}'
     ],
-    'line breaks #3': [
-      'div a{width:500px\r\n}',
-      'div a{width:500px}'
-    ],
     'line breaks with whitespace lines': [
       'div \n \t\n \na\r\n, p { width:500px }',
       'div a,p{width:500px}'

diff --git a/test/selectors/tokenizer-test.js b/test/selectors/tokenizer-test.js
index cdcacdc..b5e1130 100644 (file)
--- a/test/selectors/tokenizer-test.js
+++ b/test/selectors/tokenizer-test.js
@@ -61,7 +61,7 @@ vows.describe(Tokenizer)
         }]
       ],
       'a selector with whitespace': [
-        'a {color:red;\n\ndisplay :  block }',
+        'a {color:red;\n\ndisplay :\r\n  block }',
         [{
           kind: 'selector',
           value: [{ value: 'a' }],
@@ -71,6 +71,10 @@ vows.describe(Tokenizer)
           }]
         }]
       ],
+      'a selector with suffix whitespace': [
+        'div a{color:red\r\n}',
+        [{ kind: 'selector', value: [{ value: 'div a' }], body: [{ value: 'color:red' }] }]
+      ],
       'a selector with whitespace in functions': [
         'a{color:rgba( 255, 255, 0, 0.5  )}',
         [{