Fixes #403 - reworks tracking input files in source maps.

* We don't keep filenames in escaped data anymore.

diff --git a/History.md b/History.md
index 68cc393..0953d74 100644 (file)
--- a/History.md
+++ b/History.md
@@ -23,6 +23,7 @@
 * Fixed issue [#363](https://github.com/GoalSmashers/clean-css/issues/363) - `rem` units overriding `px`.
 * Fixed issue [#373](https://github.com/GoalSmashers/clean-css/issues/373) - proper background shorthand merging.
 * Fixed issue [#395](https://github.com/GoalSmashers/clean-css/issues/395) - unescaped brackets in data URIs.
+* Fixed issue [#403](https://github.com/GoalSmashers/clean-css/issues/403) - tracking input files in source maps.
 
 [2.2.19 / 2014-11-20](https://github.com/jakubpawlowicz/clean-css/compare/v2.2.18...v2.2.19)
 ==================

diff --git a/lib/clean.js b/lib/clean.js
index dbb02ae..5a739c7 100644 (file)
--- a/lib/clean.js
+++ b/lib/clean.js
@@ -18,6 +18,7 @@ var UrlsProcessor = require('./text/urls-processor');
 
 var Compatibility = require('./utils/compatibility');
 var InputSourceMapTracker = require('./utils/input-source-map-tracker');
+var SourceTracker = require('./utils/source-tracker');
 
 var CleanCSS = module.exports = function CleanCSS(options) {
   options = options || {};
@@ -45,7 +46,8 @@ var CleanCSS = module.exports = function CleanCSS(options) {
   this.context = {
     errors: [],
     warnings: [],
-    debug: options.debug
+    debug: options.debug,
+    sourceTracker: new SourceTracker()
   };
   this.errors = this.context.errors;
   this.warnings = this.context.warnings;
@@ -100,7 +102,7 @@ function runMinifier(callback, self) {
 
 function minifyWithDebug(self, data) {
   var startedAt = process.hrtime();
-  self.stats.originalSize = data.replace(/__ESCAPED_SOURCE_CLEAN_CSS\(.+\)__/g, '').replace(/__ESCAPED_SOURCE_END_CLEAN_CSS__/g, '').length;
+  self.stats.originalSize = self.context.sourceTracker.removeAll(data).length;
 
   data = minify.call(self, data);
 

diff --git a/lib/imports/inliner.js b/lib/imports/inliner.js
index ddf84f9..8c135f6 100644 (file)
--- a/lib/imports/inliner.js
+++ b/lib/imports/inliner.js
@@ -28,12 +28,6 @@ function rebaseMap(data, source) {
   });
 }
 
-function wrap(data, source) {
-  return '__ESCAPED_SOURCE_CLEAN_CSS(' + source + ')__' +
-    data +
-    '__ESCAPED_SOURCE_END_CLEAN_CSS__';
-}
-
 module.exports = function Inliner(context, options, rebase) {
   var defaultOptions = {
     timeout: 5000,
@@ -266,7 +260,8 @@ module.exports = function Inliner(context, options, rebase) {
         var importedData = chunks.join('');
         if (options.rebase)
           importedData = UrlRewriter.process(importedData, { toBase: importedUrl });
-        importedData = rebaseMap(wrap(importedData, importedUrl), importedUrl);
+        importedData = context.sourceTracker.store(importedUrl, importedData);
+        importedData = rebaseMap(importedData, importedUrl);
 
         if (mediaQuery.length > 0)
           importedData = '@media ' + mediaQuery + '{' + importedData + '}';
@@ -320,7 +315,7 @@ module.exports = function Inliner(context, options, rebase) {
         toBase: options._baseRelativeTo
       });
     }
-    importedData = wrap(importedData, path.resolve(options.relativeTo, fullPath));
+    importedData = context.sourceTracker.store(path.resolve(options.relativeTo, fullPath), importedData);
 
     if (mediaQuery.length > 0)
       importedData = '@media ' + mediaQuery + '{' + importedData + '}';

diff --git a/lib/selectors/tokenizer.js b/lib/selectors/tokenizer.js
index d8368a2..fc4463a 100644 (file)
--- a/lib/selectors/tokenizer.js
+++ b/lib/selectors/tokenizer.js
@@ -167,8 +167,8 @@ function tokenize(context) {
     } else if (what == 'escape') {
       nextEnd = chunk.indexOf('__', nextSpecial + 1);
       var escaped = chunk.substring(context.cursor, nextEnd + 2);
-      var isStartSourceMarker = escaped.indexOf('__ESCAPED_SOURCE_CLEAN_CSS') > -1;
-      var isEndSourceMarker = escaped.indexOf('__ESCAPED_SOURCE_END_CLEAN_CSS') > -1;
+      var isStartSourceMarker = !!context.outer.sourceTracker.nextStart(escaped);
+      var isEndSourceMarker = !!context.outer.sourceTracker.nextEnd(escaped);
 
       if (isStartSourceMarker) {
         if (addSourceMap)
@@ -179,7 +179,7 @@ function tokenize(context) {
           line: context.line,
           column: context.column
         });
-        context.source = escaped.substring(escaped.indexOf('(') + 1, escaped.indexOf(')'));
+        context.source = context.outer.sourceTracker.nextStart(escaped).filename;
         context.line = 1;
         context.column = 0;
       } else if (isEndSourceMarker) {

diff --git a/lib/utils/input-source-map-tracker.js b/lib/utils/input-source-map-tracker.js
index e6ce13e..d6c1560 100644 (file)
--- a/lib/utils/input-source-map-tracker.js
+++ b/lib/utils/input-source-map-tracker.js
@@ -6,8 +6,6 @@ var http = require('http');
 var https = require('https');
 var url = require('url');
 
-var SOURCE_MARKER_START = /__ESCAPED_SOURCE_CLEAN_CSS\(([^~][^\)]+)\)__/;
-var SOURCE_MARKER_END = /__ESCAPED_SOURCE_END_CLEAN_CSS__/;
 var MAP_MARKER = /\/\*# sourceMappingURL=(\S+) \*\//;
 
 var DEFAULT_TIMEOUT = 5000;
@@ -16,6 +14,7 @@ var DEFAULT_TIMEOUT = 5000;
 function InputSourceMapStore(options, outerContext) {
   this.options = options;
   this.errors = outerContext.errors;
+  this.sourceTracker = outerContext.sourceTracker;
   this.timeout = (options.inliner && options.inliner.timeout) || DEFAULT_TIMEOUT;
   this.requestOptions = (options.inliner && options.inliner.request) || {};
 
@@ -48,9 +47,10 @@ function fromSource(self, data, whenDone, context) {
   while (context.cursor < data.length) {
     var fragment = data.substring(context.cursor);
 
-    var markerStartMatch = SOURCE_MARKER_START.exec(fragment) || { index: -1 };
-    var markerEndMatch = SOURCE_MARKER_END.exec(fragment) || { index: -1 };
+    var markerStartMatch = self.sourceTracker.nextStart(fragment) || { index: -1 };
+    var markerEndMatch = self.sourceTracker.nextEnd(fragment) || { index: -1 };
     var mapMatch = MAP_MARKER.exec(fragment) || { index: -1 };
+    var sourceMapFile = mapMatch[1];
 
     nextAt = data.length;
     if (markerStartMatch.index > -1)
@@ -64,18 +64,18 @@ function fromSource(self, data, whenDone, context) {
       break;
 
     if (nextAt == markerStartMatch.index) {
-      context.files.push(markerStartMatch[1]);
+      context.files.push(markerStartMatch.filename);
     } else if (nextAt == markerEndMatch.index) {
       context.files.pop();
     } else if (nextAt == mapMatch.index) {
-      var isRemote = /^https?:\/\//.test(mapMatch[1]) || /^\/\//.test(mapMatch[1]);
+      var isRemote = /^https?:\/\//.test(sourceMapFile) || /^\/\//.test(sourceMapFile);
       if (isRemote) {
-        return fetchMapFile(self, mapMatch[1], context, proceedToNext);
+        return fetchMapFile(self, sourceMapFile, context, proceedToNext);
       } else {
         var sourceFile = context.files[context.files.length - 1];
         var sourceDir = sourceFile ? path.dirname(sourceFile) : self.options.relativeTo;
 
-        var inputMapData = fs.readFileSync(path.join(sourceDir || '', mapMatch[1]), 'utf-8');
+        var inputMapData = fs.readFileSync(path.join(sourceDir || '', sourceMapFile), 'utf-8');
         self.maps[sourceFile || undefined] = new SourceMapConsumer(inputMapData);
       }
     }

diff --git a/lib/utils/source-tracker.js b/lib/utils/source-tracker.js
new file mode 100644 (file)
index 0000000..4cc9b6d
--- /dev/null
+++ b/lib/utils/source-tracker.js
@@ -0,0 +1,31 @@
+function SourceTracker() {
+  this.sources = [];
+}
+
+SourceTracker.prototype.store = function (filename, data) {
+  this.sources.push(filename);
+
+  return '__ESCAPED_SOURCE_CLEAN_CSS' + (this.sources.length - 1) + '__' +
+    data +
+    '__ESCAPED_SOURCE_END_CLEAN_CSS__';
+};
+
+SourceTracker.prototype.nextStart = function (data) {
+  var next = /__ESCAPED_SOURCE_CLEAN_CSS(\d+)__/.exec(data);
+
+  return next ?
+    { index: next.index, filename: this.sources[~~next[1]] } :
+    null;
+};
+
+SourceTracker.prototype.nextEnd = function (data) {
+  return /__ESCAPED_SOURCE_END_CLEAN_CSS__/g.exec(data);
+};
+
+SourceTracker.prototype.removeAll = function (data) {
+  return data
+    .replace(/__ESCAPED_SOURCE_CLEAN_CSS\d+__/g, '')
+    .replace(/__ESCAPED_SOURCE_END_CLEAN_CSS__/g, '');
+};
+
+module.exports = SourceTracker;

diff --git a/test/data/partials/with__double_underscore.css b/test/data/partials/with__double_underscore.css
new file mode 100644 (file)
index 0000000..b133947
--- /dev/null
+++ b/test/data/partials/with__double_underscore.css
@@ -0,0 +1,3 @@
+.one {
+  color: green;
+}

diff --git a/test/integration-test.js b/test/integration-test.js
index 556f622..8f358f1 100644 (file)
--- a/test/integration-test.js
+++ b/test/integration-test.js
@@ -1407,6 +1407,10 @@ title']{display:block}",
     'after quoted content': [
       "/*a{display:block}*/@import url(test/data/partials/one.css);",
       ".one{color:red}"
+    ],
+    'with double underscore': [
+      '@import url(test/data/partials/with__double_underscore.css);',
+      '.one{color:green}'
     ]
   }, { root: process.cwd() }),
   'malformed but still valid @import': cssContext({

diff --git a/test/selectors/tokenizer-source-maps-test.js b/test/selectors/tokenizer-source-maps-test.js
index a7db7bf..39bf5fb 100644 (file)
--- a/test/selectors/tokenizer-source-maps-test.js
+++ b/test/selectors/tokenizer-source-maps-test.js
@@ -1,6 +1,7 @@
 var vows = require('vows');
 var assert = require('assert');
 var Tokenizer = require('../../lib/selectors/tokenizer');
+var SourceTracker = require('../../lib/utils/source-tracker');
 
 function sourceMapContext(group, specs) {
   var ctx = {};
@@ -16,7 +17,9 @@ function sourceMapContext(group, specs) {
       var target = specs[test][1][i];
 
       ctx[group + ' ' + test + ' - #' + (i + 1)] = {
-        topic: new Tokenizer({}, false, true).toTokens(specs[test][0]),
+        topic: typeof specs[test][0] == 'function' ?
+          specs[test][0]() :
+          new Tokenizer({ sourceTracker: new SourceTracker() }, false, true).toTokens(specs[test][0]),
         tokenized: tokenizedContext(target, i)
       };
     }
@@ -431,7 +434,12 @@ vows.describe('source-maps/analyzer')
   .addBatch(
     sourceMapContext('sources', {
       'one': [
-        '__ESCAPED_SOURCE_CLEAN_CSS(one.css)__a{}__ESCAPED_SOURCE_END_CLEAN_CSS__',
+        function () {
+          var tracker = new SourceTracker();
+          var tokenizer = new Tokenizer({ sourceTracker: tracker }, false, true);
+          var data = tracker.store('one.css', 'a{}');
+          return tokenizer.toTokens(data);
+        },
         [{
           kind: 'selector',
           value: [{ value: 'a', metadata: { line: 1, column: 0, source: 'one.css' } }],
@@ -439,7 +447,13 @@ vows.describe('source-maps/analyzer')
         }]
       ],
       'two': [
-        '__ESCAPED_SOURCE_CLEAN_CSS(one.css)__a{}__ESCAPED_SOURCE_END_CLEAN_CSS____ESCAPED_SOURCE_CLEAN_CSS(two.css)__\na{color:red}__ESCAPED_SOURCE_END_CLEAN_CSS__',
+        function () {
+          var tracker = new SourceTracker();
+          var tokenizer = new Tokenizer({ sourceTracker: tracker }, false, true);
+          var data1 = tracker.store('one.css', 'a{}');
+          var data2 = tracker.store('two.css', '\na{color:red}');
+          return tokenizer.toTokens(data1 + data2);
+        },
         [
           {
             kind: 'selector',

diff --git a/test/selectors/tokenizer-test.js b/test/selectors/tokenizer-test.js
index dc43460..153e719 100644 (file)
--- a/test/selectors/tokenizer-test.js
+++ b/test/selectors/tokenizer-test.js
@@ -1,13 +1,14 @@
 var vows = require('vows');
 var assert = require('assert');
 var Tokenizer = require('../../lib/selectors/tokenizer');
+var SourceTracker = require('../../lib/utils/source-tracker');
 
 function tokenizerContext(name, specs, addMetadata) {
   var ctx = {};
 
   function tokenized(target) {
     return function (source) {
-      var tokenized = new Tokenizer({}, addMetadata).toTokens(source);
+      var tokenized = new Tokenizer({ sourceTracker: new SourceTracker() }, addMetadata).toTokens(source);
       assert.deepEqual(target, tokenized);
     };
   }