change token data structure

Author: welefen

lib/lang/css/lexer.js

@@ -4,10 +4,12 @@ var think = require('thinkjs-util');
 var lexer = require('../../util/lexer.js');
 var TOKEN = require('../../util/token.js');
 var config = require('./config.js');
+var baseConfig = require('../../util/config.js');
 //var util = require('./util.js');
 
 
 var atType = config.atType;
+var multiComment = baseConfig.comments[1];
 //var isAttrChar = util.isAttrChar;
 
 module.exports = think.Class(lexer, {
@@ -20,6 +22,11 @@ module.exports = think.Class(lexer, {
     parse_selector: false,
     parse_value: false,
   },
+  /**
+   * prev token type
+   * @type {Number}
+   */
+  prevTokenType: 0,
   /**
    * get next token
    * @return {void} []
@@ -59,7 +66,7 @@ module.exports = think.Class(lexer, {
    */
   getValueToken: function(){
     var ret = '', code, chr;
-    var escape = false, comment;
+    var escape = false, comments = [], comment;
     /*jshint -W084 */
     while(this.pos < this.length){
       chr = this.text[this.pos];
@@ -71,23 +78,20 @@ module.exports = think.Class(lexer, {
       }
       // ; or }
       if (code === 0x3b || code === 0x7d) {
-        if (comment) {
-          this.rollback();
-        }
         break;
       }
-      if (!this.isWhiteSpace(chr)) {
-        comment = '';
+      if (comments.length && !this.isWhiteSpace(chr) && !this.lookAt(multiComment[0])) {
+        comments = [];
       }
       if (!escape && (code === 0x22 || code === 0x27)) {
         ret += this.getQuote({
           rollback: true
         }).value;
         continue;
       }else if (code === 0x2f) {
-        this.record();
-        comment = this.getCommentToken(1, 0, false);
+        comment = this.getCommentToken(1, true, !comments.length);
         if (comment) {
+          comments.push(comment);
           continue;
         }
       }else if (code === 0x28) { // ( )
@@ -102,39 +106,39 @@ module.exports = think.Class(lexer, {
       ret += this.next();
     }
     var token = this.getToken(TOKEN.CSS_VALUE, ret);
-    token.value = this.removeRightSpace(ret);
+    token.value = this.skipRightSpace(ret);
+
+    if (comments.length) {
+      this.addComment(comments);
+    }
     return token;
   },
   /**
-   * remove right space
-   * @param  {String} value []
-   * @return {String}       []
+   * add comment to next token
+   * @param {Array} comments []
    */
-  removeRightSpace: function(value){
-    var index = value.length - 1;
-    var times = 0, chr;
-    //if last char is \n, remove it
-    //and set newlines for next token
-    while(true){
-      chr = value[index];
-      if (this.isWhiteSpace(chr)) {
-        index--;
-        if (chr === '\n') {
-          times++;
-        }
-        continue;
-      }
-      break;
+  addComment: function(comments){
+    switch(comments.length){
+      case 1:
+        this.commentBefore.push(comments[0]);
+        break;
+      case 2:
+        this.commentBefore.push(comments[0], comments[1]);
+        break;
+      case 3:
+        this.commentBefore.push(comments[0], comments[1], comments[2]);
+        break;
+      default:
+        [].push.call(this.commentBefore, comments);
+        break;
     }
-    this.newlines += times;
-    return value.slice(0, index + 1);
   },
   /**
    * get selector or name token
    * @return {Object} []
    */
   getSelectorOrPropertyToken: function(){
-    var ret = '', code, str, token, type, chr, comment;
+    var ret = '', code, str, token, type, chr, comments = [], comment;
     this.record();
     /*jshint -W084 */
     while(this.pos < this.length){
@@ -158,12 +162,12 @@ module.exports = think.Class(lexer, {
         }
         break;
       }
-      if (!this.isWhiteSpace(chr)) {
-        comment = '';
+      if (comments.length && !this.isWhiteSpace(chr) && !this.lookAt(multiComment[0])) {
+        comments = [];
       }
       if (code === 0x2f) {
         this.record();
-        comment = this.getCommentToken(1, 0, false);
+        comment = this.getCommentToken(1, true, !comments.length);
         if (comment) {
           continue;
         }
@@ -184,7 +188,11 @@ module.exports = think.Class(lexer, {
       ret += this.next();
     }
     token = this.getToken(type, ret);
-    token.value = this.removeRightSpace(ret);
+    token.value = this.skipRightSpace(ret);
+
+    if (comments.length) {
+      this.addComment(comments);
+    }
     return token;
   },
   /**
@@ -196,7 +204,7 @@ module.exports = think.Class(lexer, {
     while(this.text.charCodeAt(this.pos) === 0x2f && this.text.charCodeAt(this.pos + 1) === 0x2a){
       var comment = this.getCommentToken(1, 0);
       if (comment) {
-        this.comments.push(comment);
+        this.commentBefore.push(comment);
       }else{
         break;
       }
@@ -248,7 +256,21 @@ module.exports = think.Class(lexer, {
       ret += this.next();
     }
     var token = this.getToken(type, ret);
-    token.value = this.removeRightSpace(ret);
+    token.value = this.skipRightSpace(ret);
     return token;
+  },
+  /**
+   * run
+   * @return {Array} [text tokens]
+   */
+  run: function(options){
+    this._run(options);
+    var ret = [], token;
+    /*jshint -W084 */
+    for(;token = this.getNextToken();){
+      this.prevTokenType = token.type;
+      ret.push(token);
+    }
+    return ret;
   }
 })

lib/lang/html/compress.js

@@ -59,16 +59,6 @@ module.exports = think.Class(base, {
    * @type {Object}
    */
   prevToken: null,
-  /**
-   * space before token
-   * @type {Boolean}
-   */
-  spaceBefore: false,
-  /**
-   * space before next token
-   * @type {String}
-   */
-  spaceBeforeNextToken: false,
   /**
    * next token
    * @type {Object}
@@ -90,14 +80,13 @@ module.exports = think.Class(base, {
    */
   getToken: function(){
     if (!this.token) {
-      this.token = this.tokens[this.index];
+      this.token = this.tokens[0];
     }else{
-      this.prevToken = this.token;
       if (!this.nextToken) {
         return null;
       }
+      this.prevToken = this.token;
       this.token = this.nextToken;
-      this.spaceBefore = this.spaceBeforeNextToken;
     }
     var prevToken = this.prevToken;
     if (prevToken) {
@@ -118,18 +107,14 @@ module.exports = think.Class(base, {
       }
     }
     this.nextToken = null;
-    this.spaceBeforeNextToken = false;
     var next;
-    /*jshint -W084 */
-    for(;next = this.tokens[++this.index];){
+    while(this.index < this.length - 1){
+      next = this.tokens[++this.index];
       if (next.type === TOKEN.ILLEGAL) {
         continue;
       }
-      if (next.type !== TOKEN.HTML_TEXT || !/^\s+$/.test(next.value)) {
-        this.nextToken = next;
-        break;
-      }
-      this.spaceBeforeNextToken = true;
+      this.nextToken = next;
+      break;
     }
     return this.token;
   },
@@ -411,7 +396,7 @@ module.exports = think.Class(base, {
     var where = optionalEndTags[tag], nextToken = this.nextToken;
     // next token has comment
     if (where.next_comment === false) {
-      if (nextToken && nextToken.comments.length && !options.remove_comment) {
+      if (nextToken && nextToken.commentBefore.length && !options.remove_comment) {
         return value;
       }
     }
@@ -453,16 +438,12 @@ module.exports = think.Class(base, {
     var ret = '', options = this.options;
     // not remove comment
     if (!options.remove_comment) {
-      var comments = token.comments;
+      var comments = token.commentBefore;
       for(var i = 0, length = comments.length; i < length; i++){
         ret += comments[i].value;
       }
     }
-    if (this.lastHasSpace) {
-      return ret;
-    }
-    var space = this.spaceBefore || token.newlines;
-    if (!space) {
+    if (this.lastHasSpace || !this.token.spaceBefore) {
       return ret;
     }
     var type = token.type;
@@ -474,10 +455,6 @@ module.exports = think.Class(base, {
         type === TOKEN.TPL) {
       return ret;
     }
-    // space before style or script can removed
-    if (type === TOKEN.HTML_TAG_SCRIPT || type === TOKEN.HTML_TAG_STYLE) {
-      return ret;
-    }
     //safe tag
     if (type === TOKEN.HTML_TAG_END || type === TOKEN.HTML_TAG_START) {
       if (token._tag in safeTag) {
@@ -491,18 +468,20 @@ module.exports = think.Class(base, {
         return ret;
       }
       if (options.remove_space_in_block_tag) {
+        // www </div> => www</div>
+        if (token._tag in blockTag && prevType === TOKEN.HTML_TEXT) {
+          return ret;
+        }
         // remove space between 2 block end tags
         // </div> </div> => </div></div>
         if (prevType === TOKEN.HTML_TAG_END && type === TOKEN.HTML_TAG_END) {
-          if (blockTag[prev._tag] && blockTag[token._tag]) {
+          if (prev._tag in blockTag && token._tag in blockTag) {
             return ret;
           }
         }
         // www <div> => www<div>
-        else if (type === TOKEN.HTML_TAG_START) {
-          if (blockTag[token._tag]) {
-            return ret;
-          }
+        else if (type === TOKEN.HTML_TAG_START && token._tag in blockTag) {
+          return ret;
         }
       }
     }
@@ -515,29 +494,20 @@ module.exports = think.Class(base, {
    * @return {String}       []
    */
   compressText: function(token){
-    var value = token.value;
+    var value = token.value, space = token.spaceBefore ? ' ' : '';
     // async api may have line comment, can't compress it
     if (value.indexOf('//') > -1) {
-      return value;
+      return space + value;
     }
+    value = value.replace(/\s+/g, ' ');
     var prev = this.prevToken, options = this.options;
     //if prev token is block tag, trim it
     if (prev && (prev.type === TOKEN.HTML_TAG_START || prev.type === TOKEN.HTML_TAG_END)) {
       if (options.remove_space_in_block_tag && blockTag[prev._tag]) {
-        return value.trim();
-      }
-    }
-    if (token.newlines) {
-      value = ' ' + value;
-    }
-    value = value.replace(/\s+/g, ' ');
-    //remove space before doctype
-    if (value === ' ' && this.nextToken) {
-      var type = this.nextToken.type;
-      if (type === TOKEN.HTML_DOCTYPE || type === TOKEN.HTML_TAG_SCRIPT || type === TOKEN.HTML_TAG_STYLE) {
-        return '';
+        return value;
       }
     }
+    value = space + value;
     return value;
   },
   /**
@@ -587,12 +557,13 @@ module.exports = think.Class(base, {
    * @return {String}         []
    */
   run: function(options){
-    this.super('run', options);
+    this._run(options);
     var instance = this.getInstance(lexer, true);
     this.tokens = instance.run({
       tag_attrs: true
     });
     this.index = 0;
+    this.length = this.tokens.length;
     //don't compress xml text
     if (instance.isXML) {
       return this.text;