diff -r 06a7890f802e editor.util/manifest.mf
--- a/editor.util/manifest.mf Wed May 28 13:50:31 2008 +0200
+++ b/editor.util/manifest.mf Wed May 28 14:48:54 2008 +0200
@@ -1,5 +1,5 @@
Manifest-Version: 1.0
OpenIDE-Module: org.netbeans.modules.editor.util/1
OpenIDE-Module-Localizing-Bundle: org/netbeans/lib/editor/util/Bundle.properties
-OpenIDE-Module-Specification-Version: 1.23
+OpenIDE-Module-Specification-Version: 1.24
AutoUpdate-Show-In-Client: false
diff -r 06a7890f802e editor.util/src/org/netbeans/lib/editor/util/CharSequenceUtilities.java
--- a/editor.util/src/org/netbeans/lib/editor/util/CharSequenceUtilities.java Wed May 28 13:50:31 2008 +0200
+++ b/editor.util/src/org/netbeans/lib/editor/util/CharSequenceUtilities.java Wed May 28 14:48:54 2008 +0200
@@ -516,21 +516,34 @@
/**
* Ensure that the given start and end parameters are valid indices
* of the given text.
- * @throws IndexOutOfBoundsException if the start or end are not within bounds
- * of the given text.
+ * @param start must be >=0 and <=end.
+ * @param end must be >=start and <=textLength.
+ * @param length total length of a charsequence.
+ * @throws IndexOutOfBoundsException if the start or end are not within bounds.
*/
- public static void checkIndexesValid(CharSequence text, int start, int end) {
+ public static void checkIndexesValid(int start, int end, int length) {
if (start < 0) {
throw new IndexOutOfBoundsException("start=" + start + " < 0"); // NOI18N
}
if (end < start) {
throw new IndexOutOfBoundsException("end=" + end + " < start=" + start); // NOI18N
}
- if (end > text.length()) {
- throw new IndexOutOfBoundsException("end=" + end // NOI18N
- + " > text.length()=" + text.length()); // NOI18N
+ if (end > length) {
+ throw new IndexOutOfBoundsException("end=" + end + " > length()=" + length); // NOI18N
}
}
-
+
+ /**
+ * Ensure that the given start and end parameters are valid indices
+ * of the given text.
+ * @param text non-null char sequence.
+ * @param start must be >=0 and <=end.
+ * @param end must be >=start and <=text.length().
+ * @throws IndexOutOfBoundsException if the start or end are not within bounds
+ * of the given text.
+ */
+ public static void checkIndexesValid(CharSequence text, int start, int end) {
+ checkIndexesValid(start, end, text.length());
+ }
}
diff -r 06a7890f802e editor.util/src/org/netbeans/lib/editor/util/FlyOffsetGapList.java
--- a/editor.util/src/org/netbeans/lib/editor/util/FlyOffsetGapList.java Wed May 28 13:50:31 2008 +0200
+++ b/editor.util/src/org/netbeans/lib/editor/util/FlyOffsetGapList.java Wed May 28 14:48:54 2008 +0200
@@ -60,8 +60,13 @@
private int offsetGapLength = Integer.MAX_VALUE / 2; // 32 bytes
public FlyOffsetGapList() {
+ this(10);
}
-
+
+ public FlyOffsetGapList(int initialCapacity) {
+ super(initialCapacity);
+ }
+
/**
* Get the raw offset of the given element currently stored in the list.
*
diff -r 06a7890f802e editor.util/src/org/netbeans/lib/editor/util/OffsetGapList.java
--- a/editor.util/src/org/netbeans/lib/editor/util/OffsetGapList.java Wed May 28 13:50:31 2008 +0200
+++ b/editor.util/src/org/netbeans/lib/editor/util/OffsetGapList.java Wed May 28 14:48:54 2008 +0200
@@ -82,8 +82,13 @@
private int offsetGapLength = Integer.MAX_VALUE / 2; // 32 bytes
public OffsetGapList() {
+ this(10);
}
-
+
+ public OffsetGapList(int initialCapacity) {
+ super(initialCapacity);
+ }
+
/**
* Get the raw offset of the given element currently stored in the list.
*
diff -r 06a7890f802e editor.util/test/unit/src/org/netbeans/lib/editor/util/CharSequenceUtilitiesTest.java
--- a/editor.util/test/unit/src/org/netbeans/lib/editor/util/CharSequenceUtilitiesTest.java Wed May 28 13:50:31 2008 +0200
+++ b/editor.util/test/unit/src/org/netbeans/lib/editor/util/CharSequenceUtilitiesTest.java Wed May 28 14:48:54 2008 +0200
@@ -138,6 +138,15 @@
// endsWith
assertTrue(CharSequenceUtilities.endsWith(string, string.substring(CHARS_LENGTH - SUBSTR_LENGTH)));
+
+ CharSequenceUtilities.checkIndexesValid(0, 3, 3); // start,end,length
+ CharSequenceUtilities.checkIndexesValid(1, 3, 3);
+ try {
+ CharSequenceUtilities.checkIndexesValid(1, 4, 3);
+ TestCase.fail("IndexOutOfBoundsException was expected.");
+ } catch (IndexOutOfBoundsException e) {
+ // Expected
+ }
}
public void generateChars(char[] chars) {
diff -r 06a7890f802e lexer/apichanges.xml
--- a/lexer/apichanges.xml Wed May 28 13:50:31 2008 +0200
+++ b/lexer/apichanges.xml Wed May 28 14:48:54 2008 +0200
@@ -113,6 +113,25 @@
+
+
+ Joined Sections Lexing
+
+
+
+
+
+
+ Embeddings that request input sections to be joined before lexing
+ are now lexed as a single section.
+
+ Token.isRemoved() was added to check whether a particular token
+ is still present in token hierarchy or whether it was removed as part of a modification.
+
+
+
+
+
Lexer API Cleanup
diff -r 06a7890f802e lexer/nbproject/project.properties
--- a/lexer/nbproject/project.properties Wed May 28 13:50:31 2008 +0200
+++ b/lexer/nbproject/project.properties Wed May 28 14:48:54 2008 +0200
@@ -43,4 +43,4 @@
javadoc.arch=${basedir}/arch.xml
javadoc.apichanges=${basedir}/apichanges.xml
javadoc.docfiles=${basedir}/api/doc
-spec.version.base=1.27.0
+spec.version.base=1.28.0
diff -r 06a7890f802e lexer/nbproject/project.xml
--- a/lexer/nbproject/project.xml Wed May 28 13:50:31 2008 +0200
+++ b/lexer/nbproject/project.xml Wed May 28 14:48:54 2008 +0200
@@ -52,7 +52,7 @@
1
- 1.15
+ 1.24
diff -r 06a7890f802e lexer/src/org/netbeans/api/lexer/Token.java
--- a/lexer/src/org/netbeans/api/lexer/Token.java Wed May 28 13:50:31 2008 +0200
+++ b/lexer/src/org/netbeans/api/lexer/Token.java Wed May 28 14:48:54 2008 +0200
@@ -40,6 +40,8 @@
*/
package org.netbeans.api.lexer;
+
+import java.util.List;
/**
* Token describes a lexical element of input text.
@@ -207,12 +209,46 @@
* @return true if the token is flyweight or false otherwise.
*/
public abstract boolean isFlyweight();
+
+ /**
+ * Check whether this token is no longer part of the token hierarchy.
+ *
+ * @return true if the token was removed from the token hierarchy
+ * or false if it's still present in the hierarchy.
+ */
+ public abstract boolean isRemoved();
/**
* Check whether this token represents a complete token
- * or whether it's a part of a complete token.
+ * or whether it's a particular part of a complete token.
+ *
+ * Some lexers may also use this information to express an incomplete token.
+ * For example an unclosed block comment at the end of java source
+ * is represented as a BLOCK_COMMENT token id and {@link PartType#START}.
+ *
+ * @return {@link PartType#COMPLETE} for regular token or other part types
+ * for particular token parts.
*/
public abstract PartType partType();
+
+ /**
+ * Get a complete token that is joined from multiple parts (this token is one of those parts).
+ *
+ * @return complete token or null if this token is not a part of any token.
+ */
+ public abstract Token joinToken();
+
+ /**
+ * Get all token parts comprising this token ordered from lowest to highest part's offset.
+ *
+ * It's guaranteed that each token part is continuous in the input text
+ * (there are no gaps inside the token part's text).
+ *
+ * On the other hand there may be textual gaps between two adajcent token parts.
+ *
+ * @return list of token parts or null if the token is continuous.
+ */
+ public abstract List extends Token> joinedParts();
/**
* Quickly determine whether this token has any extra properties.
diff -r 06a7890f802e lexer/src/org/netbeans/api/lexer/TokenHierarchyEvent.java
--- a/lexer/src/org/netbeans/api/lexer/TokenHierarchyEvent.java Wed May 28 13:50:31 2008 +0200
+++ b/lexer/src/org/netbeans/api/lexer/TokenHierarchyEvent.java Wed May 28 14:48:54 2008 +0200
@@ -123,7 +123,7 @@
* if this event's type is not {@link TokenHierarchyEventType#MODIFICATION}.
*/
public int modificationOffset() {
- return info.modificationOffset();
+ return info.modOffset();
}
/**
diff -r 06a7890f802e lexer/src/org/netbeans/api/lexer/TokenSequence.java
--- a/lexer/src/org/netbeans/api/lexer/TokenSequence.java Wed May 28 13:50:31 2008 +0200
+++ b/lexer/src/org/netbeans/api/lexer/TokenSequence.java Wed May 28 14:48:54 2008 +0200
@@ -42,12 +42,14 @@
package org.netbeans.api.lexer;
import java.util.ConcurrentModificationException;
+import org.netbeans.lib.lexer.EmbeddedTokenList;
import org.netbeans.lib.lexer.EmbeddingContainer;
-import org.netbeans.lib.lexer.LexerUtilsConstants;
+import org.netbeans.lib.lexer.JoinTokenList;
import org.netbeans.lib.lexer.SubSequenceTokenList;
import org.netbeans.lib.lexer.LexerUtilsConstants;
import org.netbeans.lib.lexer.TokenList;
import org.netbeans.lib.lexer.token.AbstractToken;
+import org.netbeans.lib.lexer.TokenOrEmbedding;
/**
* Token sequence allows to iterate between tokens
@@ -246,7 +248,7 @@
public int offset() {
checkTokenNotNull();
if (tokenOffset == -1) {
- tokenOffset = tokenList.tokenOffset(tokenIndex);
+ tokenOffset = tokenList.tokenOffsetByIndex(tokenIndex);
}
return tokenOffset;
}
@@ -296,17 +298,7 @@
*/
public TokenSequence> embedded() {
checkTokenNotNull();
- return embeddedImpl(null);
- }
-
- private TokenSequence embeddedImpl(Language embeddedLanguage) {
- if (token.isFlyweight())
- return null;
- TokenList embeddedTokenList = LexerUtilsConstants.embeddedTokenList(
- tokenList, tokenIndex, embeddedLanguage);
- return (embeddedTokenList != null)
- ? new TokenSequence(embeddedTokenList)
- : null;
+ return embeddedImpl(null, false);
}
/**
@@ -318,7 +310,62 @@
*/
public TokenSequence embedded(Language embeddedLanguage) {
checkTokenNotNull();
- return embeddedImpl(embeddedLanguage);
+ return embeddedImpl(embeddedLanguage, false);
+ }
+
+ /**
+ * Get embedded token sequence that possibly joins multiple embeddings
+ * with the same language paths (if the embeddings allow it - see
+ * {@link LanguageEmbedding#joinSections()}) into a single input text
+ * which is then lexed as a single continuous text.
+ *
+ * If any of the resulting tokens crosses embedding's boundaries then the token
+ * is split into multiple part tokens.
+ *
+ * If the embedding does not join sections then this method behaves
+ * like {@link #embedded()}.
+ *
+ * @return embedded sequence or null if no embedding exists for this token.
+ * The token sequence will be positioned before first token of this embedding
+ * or to a join token in case the first token of this embedding is part of the join token.
+ */
+ public TokenSequence> embeddedJoined() {
+ checkTokenNotNull();
+ return embeddedImpl(null, true);
+ }
+
+ /**
+ * Get embedded token sequence if the token
+ * to which this token sequence is currently positioned
+ * has a language embedding.
+ *
+ * @throws IllegalStateException if {@link #token()} returns null.
+ */
+ public TokenSequence embeddedJoined(Language embeddedLanguage) {
+ checkTokenNotNull();
+ return embeddedImpl(embeddedLanguage, true);
+ }
+
+ private TokenSequence embeddedImpl(Language embeddedLanguage, boolean joined) {
+ if (token.isFlyweight())
+ return null;
+
+ EmbeddedTokenList embeddedTokenList
+ = EmbeddingContainer.embeddedTokenList(tokenList, tokenIndex, embeddedLanguage, true);
+ if (embeddedTokenList != null) {
+ embeddedTokenList.embeddingContainer().updateStatus();
+ TokenSequence tse;
+ JoinTokenList joinTokenList;
+ if (joined && (joinTokenList = embeddedTokenList.joinTokenList()) != null) {
+ tse = new TokenSequence(joinTokenList);
+ // Position to this etl's index
+ tse.moveIndex(joinTokenList.activeStartJoinIndex());
+ } else { // Request regular TS or no joining available
+ tse = new TokenSequence(embeddedTokenList);
+ }
+ return tse;
+ }
+ return null;
}
/**
@@ -402,10 +449,10 @@
checkModCount();
if (token != null) // Token already fetched
tokenIndex++;
- Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(tokenIndex);
- if (tokenOrEmbeddingContainer != null) {
- AbstractToken origToken = token;
- token = LexerUtilsConstants.token(tokenOrEmbeddingContainer);
+ TokenOrEmbedding tokenOrEmbedding = tokenList.tokenOrEmbedding(tokenIndex);
+ if (tokenOrEmbedding != null) { // Might be null if no more tokens available
+ AbstractToken origToken = token;
+ token = tokenOrEmbedding.token();
// If origToken == null then the right offset might already be pre-computed from move()
if (tokenOffset != -1) {
if (origToken != null) {
@@ -446,9 +493,9 @@
public boolean movePrevious() {
checkModCount();
if (tokenIndex > 0) {
- AbstractToken origToken = token;
+ AbstractToken origToken = token;
tokenIndex--;
- token = LexerUtilsConstants.token(tokenList.tokenOrEmbeddingContainer(tokenIndex));
+ token = tokenList.tokenOrEmbedding(tokenIndex).token();
if (tokenOffset != -1) {
// If the token list is continuous or the original token
// is flyweight (there cannot be a gap before flyweight token)
@@ -501,13 +548,15 @@
public int moveIndex(int index) {
checkModCount();
if (index >= 0) {
- Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(index);
- if (tokenOrEmbeddingContainer != null) { // enough tokens
- resetTokenIndex(index);
- } else // Token at the requested index does not exist - leave orig. index
- resetTokenIndex(tokenCount());
- } else // index < 0
- resetTokenIndex(0);
+ TokenOrEmbedding tokenOrEmbedding = tokenList.tokenOrEmbedding(index);
+ if (tokenOrEmbedding != null) { // enough tokens
+ resetTokenIndex(index, -1);
+ } else {// Token at the requested index does not exist - leave orig. index
+ resetTokenIndex(tokenCount(), -1);
+ }
+ } else {// index < 0
+ resetTokenIndex(0, -1);
+ }
return index - tokenIndex;
}
@@ -555,7 +604,7 @@
*
* If token filtering is used there may be gaps that are not covered
* by any tokens and if the offset is contained in such gap then
- * the token sequence will be positioned before the token that follows the gap.
+ * the token sequence will be positioned before the token that precedes the gap.
*
*
*
@@ -563,96 +612,22 @@
* @return difference between the reqeuested offset
* and the start offset of the token
* before which the the token sequence gets positioned.
+ *
+ * If positioned right after the last token then (offset - last-token-end-offset)
+ * is returned.
*
* @throws ConcurrentModificationException if this token sequence
* is no longer valid because of an underlying mutable input source modification.
*/
public int move(int offset) {
checkModCount();
- // Token count in the list may change as possibly other threads
- // keep asking for tokens. Root token list impls create tokens lazily
- // when asked by clients.
- int tokenCount = tokenList.tokenCountCurrent(); // presently created token count
- if (tokenCount == 0) { // no tokens yet -> attempt to create at least one
- if (tokenList.tokenOrEmbeddingContainer(0) == null) { // really no tokens at all
- // In this case the token sequence could not be positioned yet
- // so no need to reset "index" or other vars
- resetTokenIndex(0);
- return offset;
- }
- // Re-get the present token count (could be created a chunk of tokens at once)
- tokenCount = tokenList.tokenCountCurrent();
+ int[] indexAndTokenOffset = tokenList.tokenIndex(offset);
+ if (indexAndTokenOffset[0] != -1) { // Valid index and token-offset
+ resetTokenIndex(indexAndTokenOffset[0], indexAndTokenOffset[1]);
+ } else { // No tokens in token list (indexAndOffset[1] == 0)
+ resetTokenIndex(0, -1); // Set Index to zero and offset to invalid
}
-
- // tokenCount surely >0
- int prevTokenOffset = tokenList.tokenOffset(tokenCount - 1);
- if (offset > prevTokenOffset) { // may need to create further tokens if they do not exist
- // Force token list to create subsequent tokens
- // Cannot subtract offset by each token's length because
- // there may be gaps between tokens due to token id filter use.
- int tokenLength = LexerUtilsConstants.token(tokenList, tokenCount - 1).length();
- while (offset >= prevTokenOffset + tokenLength) { // above present token
- Object tokenOrEmbeddingContainer = tokenList.tokenOrEmbeddingContainer(tokenCount);
- if (tokenOrEmbeddingContainer != null) {
- AbstractToken t = LexerUtilsConstants.token(tokenOrEmbeddingContainer);
- if (t.isFlyweight()) { // need to use previous tokenLength
- prevTokenOffset += tokenLength;
- } else { // non-flyweight token - retrieve offset
- prevTokenOffset = tokenList.tokenOffset(tokenCount);
- }
- tokenLength = t.length();
- tokenCount++;
-
- } else { // no more tokens => position behind last token
- resetTokenIndex(tokenCount);
- tokenOffset = prevTokenOffset + tokenLength; // May assign the token's offset in advance
- return offset - tokenOffset;
- }
- }
- resetTokenIndex(tokenCount - 1);
- tokenOffset = prevTokenOffset; // May assign the token's offset in advance
- return offset - prevTokenOffset;
- }
-
- // The offset is within the currently recognized tokens
- // Use binary search
- int low = 0;
- int high = tokenCount - 1;
-
- while (low <= high) {
- int mid = (low + high) / 2;
- int midStartOffset = tokenList.tokenOffset(mid);
-
- if (midStartOffset < offset) {
- low = mid + 1;
- } else if (midStartOffset > offset) {
- high = mid - 1;
- } else {
- // Token starting exactly at offset found
- resetTokenIndex(mid);
- tokenOffset = midStartOffset;
- return 0; // right at the token begining
- }
- }
-
- // Not found exactly and high + 1 == low => high < low
- // BTW there may be gaps between tokens; if offset is in gap then position to higher token
- if (high >= 0) { // could be -1
- AbstractToken t = LexerUtilsConstants.token(tokenList, high);
- prevTokenOffset = tokenList.tokenOffset(high);
- // If gaps allowed check whether the token at "high" contains the offset
- if (!tokenList.isContinuous() && offset > prevTokenOffset + t.length()) {
- // Offset in the gap above the "high" token
- high++;
- prevTokenOffset += t.length();
- }
- } else { // at least one token exists => use token at index 0
- high = 0;
- prevTokenOffset = tokenList.tokenOffset(0); // result may differ from 0
- }
- resetTokenIndex(high);
- tokenOffset = prevTokenOffset;
- return offset - prevTokenOffset;
+ return offset - indexAndTokenOffset[1];
}
/**
@@ -663,7 +638,7 @@
* @see #tokenCount()
*/
public boolean isEmpty() {
- return (tokenIndex == 0 && tokenList.tokenOrEmbeddingContainer(0) == null);
+ return (tokenIndex == 0 && tokenList.tokenOrEmbedding(0) == null);
}
/**
@@ -713,8 +688,9 @@
tl = stl.delegate();
startOffset = Math.max(startOffset, stl.limitStartOffset());
endOffset = Math.min(endOffset, stl.limitEndOffset());
- } else // Regular token list
+ } else {// Regular token list
tl = tokenList;
+ }
return new TokenSequence(new SubSequenceTokenList(tl, startOffset, endOffset));
}
@@ -733,14 +709,14 @@
@Override
public String toString() {
return LexerUtilsConstants.appendTokenList(null, tokenList,
- tokenIndex, 0, Integer.MAX_VALUE, true, 0).toString();
+ tokenIndex, 0, Integer.MAX_VALUE, true, 0, true).toString();
}
- private void resetTokenIndex(int index) {
+ private void resetTokenIndex(int index, int offset) {
// Position to the given index e.g. by move() and moveIndex()
tokenIndex = index;
token = null;
- tokenOffset = -1;
+ tokenOffset = offset;
}
private void checkTokenNotNull() {
diff -r 06a7890f802e lexer/src/org/netbeans/lib/lexer/BatchTokenList.java
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/lexer/src/org/netbeans/lib/lexer/BatchTokenList.java Wed May 28 14:48:54 2008 +0200
@@ -0,0 +1,247 @@
+/*
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ *
+ * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
+ *
+ * The contents of this file are subject to the terms of either the GNU
+ * General Public License Version 2 only ("GPL") or the Common
+ * Development and Distribution License("CDDL") (collectively, the
+ * "License"). You may not use this file except in compliance with the
+ * License. You can obtain a copy of the License at
+ * http://www.netbeans.org/cddl-gplv2.html
+ * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
+ * specific language governing permissions and limitations under the
+ * License. When distributing the software, include this License Header
+ * Notice in each file and include the License file at
+ * nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Sun in the GPL Version 2 section of the License file that
+ * accompanied this code. If applicable, add the following below the
+ * License Header, with the fields enclosed by brackets [] replaced by
+ * your own identifying information:
+ * "Portions Copyrighted [year] [name of copyright owner]"
+ *
+ * Contributor(s):
+ *
+ * The Original Software is NetBeans. The Initial Developer of the Original
+ * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
+ * Microsystems, Inc. All Rights Reserved.
+ *
+ * If you wish your version of this file to be governed by only the CDDL
+ * or only the GPL Version 2, indicate your decision by adding
+ * "[Contributor] elects to include this software in this distribution
+ * under the [CDDL or GPL Version 2] license." If you do not indicate a
+ * single choice of license, a recipient has the option to distribute
+ * your version of this file under either the CDDL, the GPL Version 2 or
+ * to extend the choice of license to its licensees as provided above.
+ * However, if you add GPL Version 2 code and therefore, elected the GPL
+ * Version 2 license, then the option applies only if the new code is
+ * made subject to such option by the copyright holder.
+ */
+
+package org.netbeans.lib.lexer;
+
+import java.util.ArrayList;
+import java.util.Set;
+import org.netbeans.api.lexer.Language;
+import org.netbeans.api.lexer.LanguagePath;
+import org.netbeans.api.lexer.InputAttributes;
+import org.netbeans.api.lexer.TokenId;
+import org.netbeans.lib.lexer.token.AbstractToken;
+import org.netbeans.lib.lexer.token.TextToken;
+
+
+/**
+ * Token list used for immutable inputs.
+ *
+ * @author Miloslav Metelka
+ * @version 1.00
+ */
+
+public final class BatchTokenList
+extends ArrayList> implements TokenList {
+
+ /** Flag for additional correctness checks (may degrade performance). */
+ private static final boolean testing = Boolean.getBoolean("netbeans.debug.lexer.test");
+
+ private static boolean maintainLAState;
+
+ /**
+ * Check whether lookaheads and states are stored for testing purposes.
+ */
+ public static boolean isMaintainLAState() {
+ return maintainLAState;
+ }
+
+ public static void setMaintainLAState(boolean maintainLAState) {
+ BatchTokenList.maintainLAState = maintainLAState;
+ }
+
+ private final TokenHierarchyOperation,T> tokenHierarchyOperation;
+
+ private final CharSequence inputSourceText;
+
+ private final LanguagePath languagePath;
+
+ private final Set skipTokenIds;
+
+ private final InputAttributes inputAttributes;
+
+ /**
+ * Lexer input used for lexing of the input.
+ */
+ private LexerInputOperation lexerInputOperation;
+
+ private LAState laState;
+
+
+ public BatchTokenList(TokenHierarchyOperation,T> tokenHierarchyOperation, CharSequence inputText,
+ Language language, Set skipTokenIds, InputAttributes inputAttributes) {
+ this.tokenHierarchyOperation = tokenHierarchyOperation;
+ this.inputSourceText = inputText;
+ this.languagePath = LanguagePath.get(language);
+ this.skipTokenIds = skipTokenIds;
+ this.inputAttributes = inputAttributes;
+ if (testing) { // Maintain lookaheads and states when in test environment
+ laState = LAState.empty();
+ }
+ this.lexerInputOperation = createLexerInputOperation();
+ }
+
+ protected LexerInputOperation createLexerInputOperation() {
+ return new TextLexerInputOperation(this);
+ }
+
+ public TokenList> rootTokenList() {
+ return this; // this list should always be the root list of the token hierarchy
+ }
+
+ public CharSequence inputSourceText() {
+ return inputSourceText;
+ }
+
+ public TokenHierarchyOperation,?> tokenHierarchyOperation() {
+ return tokenHierarchyOperation;
+ }
+
+ public LanguagePath languagePath() {
+ return languagePath;
+ }
+
+ public synchronized int tokenCount() {
+ if (lexerInputOperation != null) { // still lexing
+ tokenOrEmbeddingImpl(Integer.MAX_VALUE);
+ }
+ return size();
+ }
+
+ public int tokenCountCurrent() {
+ return size();
+ }
+
+ public int tokenOffset(AbstractToken token) {
+ int rawOffset = token.rawOffset();
+ // Children offsets should be absolute
+ return rawOffset;
+ }
+
+ public int tokenOffsetByIndex(int index) {
+ AbstractToken token = existingToken(index);
+ int offset;
+ if (token.isFlyweight()) {
+ offset = 0;
+ while (--index >= 0) {
+ token = existingToken(index);
+ offset += token.length();
+ if (!token.isFlyweight()) {
+ offset += token.offset(null);
+ break;
+ }
+ }
+ } else { // non-flyweight offset
+ offset = token.offset(null);
+ }
+ return offset;
+ }
+
+ public int[] tokenIndex(int offset) {
+ return LexerUtilsConstants.tokenIndexLazyTokenCreation(this, offset);
+ }
+
+ public synchronized TokenOrEmbedding tokenOrEmbedding(int index) {
+ return tokenOrEmbeddingImpl(index);
+ }
+
+ private TokenOrEmbedding tokenOrEmbeddingImpl(int index) {
+ while (lexerInputOperation != null && index >= size()) {
+ AbstractToken token = lexerInputOperation.nextToken();
+ if (token != null) { // lexer returned valid token
+ add(token);
+ if (laState != null) { // maintaining lookaheads and states
+ laState = laState.add(lexerInputOperation.lookahead(),
+ lexerInputOperation.lexerState());
+ }
+ } else { // no more tokens from lexer
+ lexerInputOperation.release();
+ lexerInputOperation = null;
+ trimToSize();
+ }
+ }
+ return (index < size()) ? get(index) : null;
+ }
+
+ private AbstractToken existingToken(int index) {
+ return get(index).token();
+ }
+
+ public int lookahead(int index) {
+ return (laState != null) ? laState.lookahead(index) : -1;
+ }
+
+ public Object state(int index) {
+ return (laState != null) ? laState.state(index) : null;
+ }
+
+ public int startOffset() {
+ return 0;
+ }
+
+ public int endOffset() {
+ int cntM1 = tokenCount() - 1;
+ if (cntM1 >= 0)
+ return tokenOffsetByIndex(cntM1) + tokenOrEmbeddingImpl(cntM1).token().length();
+ return 0;
+ }
+
+ public boolean isRemoved() {
+ return false;
+ }
+
+ public int modCount() {
+ return LexerUtilsConstants.MOD_COUNT_IMMUTABLE_INPUT; // immutable input
+ }
+
+ public synchronized AbstractToken replaceFlyToken(
+ int index, AbstractToken flyToken, int offset) {
+ TextToken nonFlyToken = ((TextToken)flyToken).createCopy(this, offset);
+ set(index, nonFlyToken);
+ return nonFlyToken;
+ }
+
+ public void wrapToken(int index, EmbeddingContainer embeddingContainer) {
+ set(index, embeddingContainer);
+ }
+
+ public InputAttributes inputAttributes() {
+ return inputAttributes;
+ }
+
+ public boolean isContinuous() {
+ return (skipTokenIds == null);
+ }
+
+ public Set skipTokenIds() {
+ return skipTokenIds;
+ }
+
+}
diff -r 06a7890f802e lexer/src/org/netbeans/lib/lexer/CharPreprocessorOperation.java
--- a/lexer/src/org/netbeans/lib/lexer/CharPreprocessorOperation.java Wed May 28 13:50:31 2008 +0200
+++ b/lexer/src/org/netbeans/lib/lexer/CharPreprocessorOperation.java Wed May 28 14:48:54 2008 +0200
@@ -241,8 +241,8 @@
public void notifyError(String errorMessage) {
if (lexerInputOperation != null) {
int parentIndex = parent.readIndex(); // Get the
- lexerInputOperation.notifyPreprocessorError(
- new CharPreprocessorError(errorMessage, parent.deepRawLength(parentIndex)));
+// lexerInputOperation.notifyPreprocessorError(
+// new CharPreprocessorError(errorMessage, parent.deepRawLength(parentIndex)));
}
}
@@ -288,10 +288,10 @@
return tokenLength;
}
- public void tokenRecognized(int tokenLength) {
+ public void assignTokenLength(int tokenLength, boolean skipToken) {
this.tokenLength = tokenLength;
// Modify tokenLength for preprocessed characters
- parent.tokenRecognized(parentLength(tokenLength));
+ parent.assignTokenLength(parentLength(tokenLength), skipToken);
}
public PreprocessedTextStorage createPreprocessedTextStorage(CharSequence rawText,
@@ -390,7 +390,7 @@
* This method is called after the token has been recognized
* to clear internal data related to processing of token's characters.
*/
- public void tokenApproved() {
+ public void consumeTokenLength() {
if (prepStartIndex != lookaheadIndex) { // some prep chars (may be after token length)
if (prepStartIndex < tokenLength) { // prep chars before token end
if (prepEndIndex <= tokenLength) { // no preprocessed chars past token end
@@ -417,7 +417,7 @@
readIndex -= tokenLength;
lookaheadIndex -= tokenLength;
- parent.tokenApproved();
+ parent.consumeTokenLength();
if (testing)
consistencyCheck();
diff -r 06a7890f802e lexer/src/org/netbeans/lib/lexer/CharProvider.java
--- a/lexer/src/org/netbeans/lib/lexer/CharProvider.java Wed May 28 13:50:31 2008 +0200
+++ b/lexer/src/org/netbeans/lib/lexer/CharProvider.java Wed May 28 14:48:54 2008 +0200
@@ -101,18 +101,19 @@
* token length in the root lexer input operation due to character
* preprocessing.
*
- * The tokenLength should be cached by this provider.
- * @param skip whether the token will be skipped due to filtering of its id.
+ * The tokenLength at a particular level should be cached by the corresponding provider.
+ *
+ * @param skipToken whether the token will be skipped due to filtering of its id.
* @return true if the token is preprocessed or false otherwise.
*/
- void tokenRecognized(int tokenLength);
+ void assignTokenLength(int tokenLength, boolean skipToken);
/**
- * Notify this provider that the token was approved and
- * that the tokenLength number of characters should be skipped
- * (tokenLength should be cached by the provider).
+ * Notify this provider that the token was created and
+ * that the tokenLength number of characters should be consumed
+ * (tokenLength should continue to be held by the provider).
*/
- void tokenApproved();
+ void consumeTokenLength();
/**
* Collect extra preprocessed characters from the parent providers.
diff -r 06a7890f802e lexer/src/org/netbeans/lib/lexer/EmbeddedJoinInfo.java
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/lexer/src/org/netbeans/lib/lexer/EmbeddedJoinInfo.java Wed May 28 14:48:54 2008 +0200
@@ -0,0 +1,122 @@
+/*
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ *
+ * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
+ *
+ * The contents of this file are subject to the terms of either the GNU
+ * General Public License Version 2 only ("GPL") or the Common
+ * Development and Distribution License("CDDL") (collectively, the
+ * "License"). You may not use this file except in compliance with the
+ * License. You can obtain a copy of the License at
+ * http://www.netbeans.org/cddl-gplv2.html
+ * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
+ * specific language governing permissions and limitations under the
+ * License. When distributing the software, include this License Header
+ * Notice in each file and include the License file at
+ * nbbuild/licenses/CDDL-GPL-2-CP. Sun designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Sun in the GPL Version 2 section of the License file that
+ * accompanied this code. If applicable, add the following below the
+ * License Header, with the fields enclosed by brackets [] replaced by
+ * your own identifying information:
+ * "Portions Copyrighted [year] [name of copyright owner]"
+ *
+ * Contributor(s):
+ *
+ * The Original Software is NetBeans. The Initial Developer of the Original
+ * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
+ * Microsystems, Inc. All Rights Reserved.
+ *
+ * If you wish your version of this file to be governed by only the CDDL
+ * or only the GPL Version 2, indicate your decision by adding
+ * "[Contributor] elects to include this software in this distribution
+ * under the [CDDL or GPL Version 2] license." If you do not indicate a
+ * single choice of license, a recipient has the option to distribute
+ * your version of this file under either the CDDL, the GPL Version 2 or
+ * to extend the choice of license to its licensees as provided above.
+ * However, if you add GPL Version 2 code and therefore, elected the GPL
+ * Version 2 license, then the option applies only if the new code is
+ * made subject to such option by the copyright holder.
+ */
+
+package org.netbeans.lib.lexer;
+
+/**
+ * Class that wraps a each embedded token list contained in join token list.
+ *
+ * @author Miloslav Metelka
+ */
+
+public final class EmbeddedJoinInfo {
+
+ public EmbeddedJoinInfo(JoinTokenListBase base, int rawJoinTokenIndex, int rawTokenListIndex) {
+ assert (base != null);
+ this.base = base;
+ this.rawJoinTokenIndex = rawJoinTokenIndex;
+ this.rawTokenListIndex = rawTokenListIndex;
+ }
+
+ /**
+ * Reference to join token list base as a join-related extension
+ * of this ETL.
+ * In fact this is the only field through which the join token list base instance
+ * is referenced.
+ */
+ public final JoinTokenListBase base; // 12 bytes (8-super + 4)
+
+ /**
+ * Index in terms of join token list
+ * that corresponds to first token of wrapped ETL.
+ *
+ * The index must be gap-preprocessed.
+ */
+ int rawJoinTokenIndex; // 16 bytes
+
+ /**
+ * Index of related ETL in a join token list (base).
+ *
+ * The index must be gap-preprocessed.
+ */
+ int rawTokenListIndex; // 20 bytes
+
+ /**
+ * Number of items to go forward to reach last part of a join token.
+ * Zero otherwise.
+ */
+ private int joinTokenLastPartShift; // 24 bytes
+
+ public int joinTokenIndex() {
+ return base.joinTokenIndex(rawJoinTokenIndex);
+ }
+
+ public void setRawJoinTokenIndex(int rawJoinTokenIndex) {
+ this.rawJoinTokenIndex = rawJoinTokenIndex;
+ }
+
+ public int tokenListIndex() {
+ return base.tokenListIndex(rawTokenListIndex);
+ }
+
+ public int joinTokenLastPartShift() {
+ return joinTokenLastPartShift;
+ }
+
+ public void setJoinTokenLastPartShift(int joinTokenLastPartShift) {
+ this.joinTokenLastPartShift = joinTokenLastPartShift;
+ }
+
+ public StringBuilder dumpInfo(StringBuilder sb) {
+ if (sb == null)
+ sb = new StringBuilder(70);
+ sb.append("jti=").append(joinTokenIndex());
+ sb.append(", tli=").append(tokenListIndex());
+ sb.append(", lps=").append(joinTokenLastPartShift());
+ return sb;
+ }
+
+ @Override
+ public String toString() {
+ return dumpInfo(null).toString();
+ }
+
+}
diff -r 06a7890f802e lexer/src/org/netbeans/lib/lexer/EmbeddedTokenList.java
--- a/lexer/src/org/netbeans/lib/lexer/EmbeddedTokenList.java Wed May 28 13:50:31 2008 +0200
+++ b/lexer/src/org/netbeans/lib/lexer/EmbeddedTokenList.java Wed May 28 14:48:54 2008 +0200
@@ -47,11 +47,11 @@
import org.netbeans.lib.editor.util.FlyOffsetGapList;
import org.netbeans.lib.lexer.inc.MutableTokenList;
import org.netbeans.api.lexer.InputAttributes;
-import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.inc.TokenListChange;
import org.netbeans.spi.lexer.LanguageEmbedding;
import org.netbeans.lib.lexer.token.AbstractToken;
+import org.netbeans.lib.lexer.token.JoinToken;
import org.netbeans.lib.lexer.token.TextToken;
@@ -73,7 +73,7 @@
*/
public final class EmbeddedTokenList
-extends FlyOffsetGapList