token,
- TokenPropertyProvider propertyProvider, Object key, Object tokenStoreValue) {
- Object tokenStoreKey = propertyProvider.tokenStoreKey();
- if (tokenStoreKey != null && tokenStoreKey.equals(key)) { // token store value
- return propertyProvider.getValue(token, tokenStoreKey, tokenStoreValue);
- } else {
- return propertyProvider.getValue(token, key);
- }
- }
-
public static String idToString(TokenId id) {
return id.name() + '[' + id.ordinal() + ']'; // NOI18N;
}
Index: lexer/src/org/netbeans/lib/lexer/token/AbstractToken.java
===================================================================
RCS file: /cvs/lexer/src/org/netbeans/lib/lexer/token/AbstractToken.java,v
retrieving revision 1.4
diff -u -r1.4 AbstractToken.java
--- lexer/src/org/netbeans/lib/lexer/token/AbstractToken.java 2 Feb 2007 14:47:28 -0000 1.4
+++ lexer/src/org/netbeans/lib/lexer/token/AbstractToken.java 3 Mar 2007 19:20:46 -0000
@@ -19,6 +19,7 @@
package org.netbeans.lib.lexer.token;
+import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenHierarchy;
import org.netbeans.api.lexer.TokenId;
@@ -63,6 +64,7 @@
*
* @return non-null identification of this token.
*/
+ @Override
public final T id() {
return id;
}
@@ -70,6 +72,7 @@
/**
* Get text represented by this token.
*/
+ @Override
public CharSequence text() {
if (tokenList != null) {
if (tokenList.getClass() == EmbeddedTokenList.class) {
@@ -113,6 +116,7 @@
this.rawOffset = rawOffset;
}
+ @Override
public final boolean isFlyweight() {
return (rawOffset == -1);
}
@@ -121,10 +125,17 @@
setRawOffset(-1);
}
+ @Override
+ public PartType partType() {
+ return PartType.COMPLETE;
+ }
+
+ @Override
public boolean isCustomText() {
return false;
}
+ @Override
public final int offset(TokenHierarchy tokenHierarchy) {
if (rawOffset == -1) { // flyweight token
return -1;
@@ -140,26 +151,32 @@
}
}
+ @Override
public boolean isPreprocessedText() {
return false;
}
+ @Override
public CharSequence preprocessedText() {
return null;
}
+ @Override
public String preprocessError() {
return null;
}
+ @Override
public int preprocessErrorIndex() {
return -1;
}
+ @Override
public boolean hasProperties() {
return false;
}
+ @Override
public Object getProperty(Object key) {
return null;
}
Index: lexer/src/org/netbeans/lib/lexer/token/ComplexToken.java
===================================================================
RCS file: lexer/src/org/netbeans/lib/lexer/token/ComplexToken.java
diff -N lexer/src/org/netbeans/lib/lexer/token/ComplexToken.java
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ lexer/src/org/netbeans/lib/lexer/token/ComplexToken.java 3 Mar 2007 19:20:46 -0000
@@ -0,0 +1,83 @@
+/*
+ * The contents of this file are subject to the terms of the Common Development
+ * and Distribution License (the License). You may not use this file except in
+ * compliance with the License.
+ *
+ * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
+ * or http://www.netbeans.org/cddl.txt.
+ *
+ * When distributing Covered Code, include this CDDL Header Notice in each file
+ * and include the License file at http://www.netbeans.org/cddl.txt.
+ * If applicable, add the following below the CDDL Header, with the fields
+ * enclosed by brackets [] replaced by your own identifying information:
+ * "Portions Copyrighted [year] [name of copyright owner]"
+ *
+ * The Original Software is NetBeans. The Initial Developer of the Original
+ * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
+ * Microsystems, Inc. All Rights Reserved.
+ */
+
+package org.netbeans.lib.lexer.token;
+
+import org.netbeans.api.lexer.PartType;
+import org.netbeans.api.lexer.TokenId;
+import org.netbeans.lib.lexer.LexerUtilsConstants;
+import org.netbeans.lib.lexer.PreprocessedTextStorage;
+import org.netbeans.spi.lexer.CharPreprocessor;
+import org.netbeans.spi.lexer.TokenPropertyProvider;
+
+/**
+ * Token that holds information about preprocessed characters
+ * and also carries properties.
+ *
+ *
+ * Instances of this token are more costly than other token types
+ * because in addition to regular information they store preprocessed
+ * text of the token.
+ *
+ * @author Miloslav Metelka
+ * @version 1.00
+ */
+
+public final class ComplexToken extends PreprocessedTextToken {
+
+ private final TokenPropertyProvider propertyProvider; // 36 bytes (32-super + 4)
+
+ private final CharSequence customText; // 40 bytes
+
+ private final PartType partType; // 44 bytes
+
+ public ComplexToken(T id, int length,
+ TokenPropertyProvider propertyProvider, CharSequence customText, PartType partType) {
+ super(id, length);
+ this.propertyProvider = propertyProvider;
+ this.customText = customText;
+ this.partType = partType;
+ }
+
+ @Override
+ public boolean hasProperties() {
+ return (propertyProvider != null);
+ }
+
+ @Override
+ public Object getProperty(Object key) {
+ return (propertyProvider != null) ? propertyProvider.getValue(this, key) : null;
+ }
+
+ @Override
+ public CharSequence text() {
+ return (customText != null) ? customText : super.text();
+ }
+
+ @Override
+ public PartType partType() {
+ return partType;
+ }
+
+ @Override
+ protected String dumpInfoTokenType() {
+ return "PPrT"; // NOI18N "PrepToken"
+ }
+
+}
Index: lexer/src/org/netbeans/lib/lexer/token/CustomTextToken.java
===================================================================
RCS file: /cvs/lexer/src/org/netbeans/lib/lexer/token/CustomTextToken.java,v
retrieving revision 1.4
diff -u -r1.4 CustomTextToken.java
--- lexer/src/org/netbeans/lib/lexer/token/CustomTextToken.java 2 Feb 2007 14:47:28 -0000 1.4
+++ lexer/src/org/netbeans/lib/lexer/token/CustomTextToken.java 3 Mar 2007 19:20:46 -0000
@@ -19,6 +19,7 @@
package org.netbeans.lib.lexer.token;
+import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.TokenId;
/**
@@ -32,24 +33,29 @@
*/
public class CustomTextToken extends DefaultToken {
-
+
private final CharSequence text; // 28 bytes (24-super + 4)
-
+
+ private final PartType partType; // 32 bytes
+
/**
* @param id non-null identification of the token.
* @param length length of the token.
* @param text non-null text of the token.
*/
- public CustomTextToken(T id, int length, CharSequence text) {
+ public CustomTextToken(T id, int length, CharSequence text, PartType partType) {
super(id, length);
assert (text != null);
this.text = text;
+ this.partType = partType;
}
-
+
+ @Override
public final CharSequence text() {
return text;
}
+ @Override
protected String dumpInfoTokenType() {
return "CusT"; // NOI18N "TextToken" or "FlyToken"
}
Index: lexer/src/org/netbeans/lib/lexer/token/DefaultToken.java
===================================================================
RCS file: /cvs/lexer/src/org/netbeans/lib/lexer/token/DefaultToken.java,v
retrieving revision 1.5
diff -u -r1.5 DefaultToken.java
--- lexer/src/org/netbeans/lib/lexer/token/DefaultToken.java 2 Feb 2007 14:47:28 -0000 1.5
+++ lexer/src/org/netbeans/lib/lexer/token/DefaultToken.java 3 Mar 2007 19:20:46 -0000
@@ -58,10 +58,12 @@
this.length = 0;
}
+ @Override
public final int length() {
return length;
}
+ @Override
protected String dumpInfoTokenType() {
return "DefT"; // NOI18N "TextToken" or "FlyToken"
}
Index: lexer/src/org/netbeans/lib/lexer/token/PreprocessedTextToken.java
===================================================================
RCS file: /cvs/lexer/src/org/netbeans/lib/lexer/token/PreprocessedTextToken.java,v
retrieving revision 1.3
diff -u -r1.3 PreprocessedTextToken.java
--- lexer/src/org/netbeans/lib/lexer/token/PreprocessedTextToken.java 2 Feb 2007 14:47:29 -0000 1.3
+++ lexer/src/org/netbeans/lib/lexer/token/PreprocessedTextToken.java 3 Mar 2007 19:20:46 -0000
@@ -50,22 +50,27 @@
this.prepError = prepError;
}
+ @Override
public synchronized CharSequence preprocessedText() {
return prepTextStorage;
}
+ @Override
public synchronized boolean isPreprocessedText() {
- return true;
+ return (prepTextStorage != null);
}
+ @Override
public String preprocessError() {
return (prepError != null) ? prepError.message() : null;
}
+ @Override
public int preprocessErrorIndex() {
return (prepError != null) ? prepError.index() : -1;
}
+ @Override
protected String dumpInfoTokenType() {
return "PreT"; // NOI18N "PrepToken"
}
Index: lexer/src/org/netbeans/lib/lexer/token/PropertyCustomTextToken.java
===================================================================
RCS file: lexer/src/org/netbeans/lib/lexer/token/PropertyCustomTextToken.java
diff -N lexer/src/org/netbeans/lib/lexer/token/PropertyCustomTextToken.java
--- lexer/src/org/netbeans/lib/lexer/token/PropertyCustomTextToken.java 2 Feb 2007 14:47:29 -0000 1.3
+++ /dev/null 1 Jan 1970 00:00:00 -0000
@@ -1,90 +0,0 @@
-/*
- * The contents of this file are subject to the terms of the Common Development
- * and Distribution License (the License). You may not use this file except in
- * compliance with the License.
- *
- * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
- * or http://www.netbeans.org/cddl.txt.
- *
- * When distributing Covered Code, include this CDDL Header Notice in each file
- * and include the License file at http://www.netbeans.org/cddl.txt.
- * If applicable, add the following below the CDDL Header, with the fields
- * enclosed by brackets [] replaced by your own identifying information:
- * "Portions Copyrighted [year] [name of copyright owner]"
- *
- * The Original Software is NetBeans. The Initial Developer of the Original
- * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
- * Microsystems, Inc. All Rights Reserved.
- */
-
-package org.netbeans.lib.lexer.token;
-
-import org.netbeans.api.lexer.TokenId;
-import org.netbeans.lib.lexer.CharPreprocessorError;
-import org.netbeans.lib.lexer.LexerUtilsConstants;
-import org.netbeans.lib.lexer.PreprocessedTextStorage;
-import org.netbeans.spi.lexer.TokenPropertyProvider;
-
-/**
- * Token that holds information about preprocessed characters.
- *
- *
- * Instances of this token are more costly than other token types
- * because in addition to regular information they store preprocessed
- * text of the token.
- *
- * @author Miloslav Metelka
- * @version 1.00
- */
-
-public final class PropertyCustomTextToken extends CustomTextToken {
-
- private TokenPropertyProvider propertyProvider; // 36 bytes (32-super + 4)
-
- private Object tokenStoreValue; // 40 bytes
-
- private PreprocessedTextStorage prepTextStorage; // 44 bytes
-
- private CharPreprocessorError prepError; // 48 bytes
-
- public PropertyCustomTextToken(T id, int length, CharSequence text,
- TokenPropertyProvider propertyProvider, Object tokenStoreValue) {
- super(id, length, text);
- this.propertyProvider = propertyProvider;
- this.tokenStoreValue = tokenStoreValue;
- }
-
- public void initPrep(PreprocessedTextStorage prepTextStorage, CharPreprocessorError prepError) {
- this.prepTextStorage = prepTextStorage;
- this.prepError = prepError;
- }
-
- public synchronized CharSequence preprocessedText() {
- return prepTextStorage;
- }
-
- public synchronized boolean isPreprocessedText() {
- return true;
- }
-
- public String preprocessError() {
- return (prepError != null) ? prepError.message() : null;
- }
-
- public int preprocessErrorIndex() {
- return (prepError != null) ? prepError.index() : -1;
- }
-
- public boolean hasProperties() {
- return true;
- }
-
- public Object getProperty(Object key) {
- return LexerUtilsConstants.getTokenProperty(this, propertyProvider, key, tokenStoreValue);
- }
-
- protected String dumpInfoTokenType() {
- return "PPCT"; // NOI18N
- }
-
-}
Index: lexer/src/org/netbeans/lib/lexer/token/PropertyPreprocessedTextToken.java
===================================================================
RCS file: lexer/src/org/netbeans/lib/lexer/token/PropertyPreprocessedTextToken.java
diff -N lexer/src/org/netbeans/lib/lexer/token/PropertyPreprocessedTextToken.java
--- lexer/src/org/netbeans/lib/lexer/token/PropertyPreprocessedTextToken.java 2 Feb 2007 14:47:29 -0000 1.3
+++ /dev/null 1 Jan 1970 00:00:00 -0000
@@ -1,67 +0,0 @@
-/*
- * The contents of this file are subject to the terms of the Common Development
- * and Distribution License (the License). You may not use this file except in
- * compliance with the License.
- *
- * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
- * or http://www.netbeans.org/cddl.txt.
- *
- * When distributing Covered Code, include this CDDL Header Notice in each file
- * and include the License file at http://www.netbeans.org/cddl.txt.
- * If applicable, add the following below the CDDL Header, with the fields
- * enclosed by brackets [] replaced by your own identifying information:
- * "Portions Copyrighted [year] [name of copyright owner]"
- *
- * The Original Software is NetBeans. The Initial Developer of the Original
- * Software is Sun Microsystems, Inc. Portions Copyright 1997-2007 Sun
- * Microsystems, Inc. All Rights Reserved.
- */
-
-package org.netbeans.lib.lexer.token;
-
-import org.netbeans.api.lexer.TokenId;
-import org.netbeans.lib.lexer.LexerUtilsConstants;
-import org.netbeans.lib.lexer.PreprocessedTextStorage;
-import org.netbeans.spi.lexer.CharPreprocessor;
-import org.netbeans.spi.lexer.TokenPropertyProvider;
-
-/**
- * Token that holds information about preprocessed characters
- * and also carries properties.
- *
- *
- * Instances of this token are more costly than other token types
- * because in addition to regular information they store preprocessed
- * text of the token.
- *
- * @author Miloslav Metelka
- * @version 1.00
- */
-
-public final class PropertyPreprocessedTextToken extends PreprocessedTextToken {
-
- private TokenPropertyProvider propertyProvider; // 28 bytes (24-super + 4)
-
- private Object tokenStoreValue; // 32 bytes
-
- public PropertyPreprocessedTextToken(T id, int length,
- TokenPropertyProvider propertyProvider, Object tokenStoreValue) {
- super(id, length);
- this.propertyProvider = propertyProvider;
- this.tokenStoreValue = tokenStoreValue;
- }
-
- public boolean hasProperties() {
- return true;
- }
-
- public Object getProperty(Object key) {
- return LexerUtilsConstants.getTokenProperty(this, propertyProvider, key, tokenStoreValue);
- }
-
- protected String dumpInfoTokenType() {
- return "PPrT"; // NOI18N "PrepToken"
- }
-
-}
-
Index: lexer/src/org/netbeans/lib/lexer/token/PropertyToken.java
===================================================================
RCS file: /cvs/lexer/src/org/netbeans/lib/lexer/token/PropertyToken.java,v
retrieving revision 1.3
diff -u -r1.3 PropertyToken.java
--- lexer/src/org/netbeans/lib/lexer/token/PropertyToken.java 2 Feb 2007 14:47:30 -0000 1.3
+++ lexer/src/org/netbeans/lib/lexer/token/PropertyToken.java 3 Mar 2007 19:20:46 -0000
@@ -19,6 +19,8 @@
package org.netbeans.lib.lexer.token;
+import org.netbeans.api.lexer.PartType;
+import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.lexer.LexerUtilsConstants;
import org.netbeans.spi.lexer.TokenPropertyProvider;
@@ -37,27 +39,35 @@
public final class PropertyToken extends DefaultToken {
- private TokenPropertyProvider propertyProvider; // 28 bytes (24-super + 4)
+ private final TokenPropertyProvider propertyProvider; // 28 bytes (24-super + 4)
- private Object tokenStoreValue; // 32 bytes
+ private final PartType partType; // 32 bytes
public PropertyToken(T id, int length,
- TokenPropertyProvider propertyProvider, Object tokenStoreValue) {
+ TokenPropertyProvider propertyProvider, PartType partType) {
super(id, length);
this.propertyProvider = propertyProvider;
- this.tokenStoreValue = tokenStoreValue;
+ this.partType = partType;
}
+ @Override
public boolean hasProperties() {
- return true;
+ return (propertyProvider != null);
}
+ @Override
public Object getProperty(Object key) {
- return LexerUtilsConstants.getTokenProperty(this, propertyProvider, key, tokenStoreValue);
+ return (propertyProvider != null) ? propertyProvider.getValue(this, key) : null;
+ }
+
+ @Override
+ public PartType partType() {
+ return partType;
}
+ @Override
protected String dumpInfoTokenType() {
return "ProT"; // NOI18N "PrepToken"
}
-
+
}
Index: lexer/src/org/netbeans/lib/lexer/token/TextToken.java
===================================================================
RCS file: /cvs/lexer/src/org/netbeans/lib/lexer/token/TextToken.java,v
retrieving revision 1.4
diff -u -r1.4 TextToken.java
--- lexer/src/org/netbeans/lib/lexer/token/TextToken.java 2 Feb 2007 14:47:30 -0000 1.4
+++ lexer/src/org/netbeans/lib/lexer/token/TextToken.java 3 Mar 2007 19:20:46 -0000
@@ -69,10 +69,12 @@
this.text = text;
}
+ @Override
public final int length() {
return text.length();
}
+ @Override
public final CharSequence text() {
return text;
}
@@ -81,6 +83,7 @@
return new TextToken(id(), tokenList, rawOffset, text());
}
+ @Override
protected String dumpInfoTokenType() {
return isFlyweight() ? "FlyT" : "TexT"; // NOI18N "TextToken" or "FlyToken"
}
Index: lexer/src/org/netbeans/spi/lexer/TokenFactory.java
===================================================================
RCS file: /cvs/lexer/src/org/netbeans/spi/lexer/TokenFactory.java,v
retrieving revision 1.6
diff -u -r1.6 TokenFactory.java
--- lexer/src/org/netbeans/spi/lexer/TokenFactory.java 2 Feb 2007 14:47:34 -0000 1.6
+++ lexer/src/org/netbeans/spi/lexer/TokenFactory.java 3 Mar 2007 19:20:46 -0000
@@ -20,6 +20,7 @@
package org.netbeans.spi.lexer;
import java.util.Set;
+import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenId;
import org.netbeans.lib.editor.util.CharSequenceUtilities;
@@ -28,9 +29,9 @@
import org.netbeans.lib.lexer.TokenIdImpl;
import org.netbeans.lib.lexer.token.CustomTextToken;
import org.netbeans.lib.lexer.token.DefaultToken;
+import org.netbeans.lib.lexer.token.ComplexToken;
import org.netbeans.lib.lexer.token.PreprocessedTextToken;
-import org.netbeans.lib.lexer.token.PropertyCustomTextToken;
-import org.netbeans.lib.lexer.token.PropertyPreprocessedTextToken;
+import org.netbeans.lib.lexer.token.ComplexToken;
import org.netbeans.lib.lexer.token.PropertyToken;
import org.netbeans.lib.lexer.token.TextToken;
@@ -104,6 +105,39 @@
}
/**
+ * Create regular token instance with an explicit length and part type.
+ *
+ * @param id non-null token id recognized by the lexer.
+ * @param length >=0 length of the token to be created. The length must not
+ * exceed the number of characters read from the lexer input.
+ * @param partType whether this token is complete token or a part of a complete token.
+ * @return non-null regular token instance.
+ *
+ * If there were any characters preprocessed by {@link CharPreprocessor}
+ * then a special token instance will be created for it.
+ *
+ * {@link #SKIP_TOKEN} will be returned
+ * if tokens for the given token id should be skipped
+ * because of token id filter.
+ */
+ public Token createToken(T id, int length, PartType partType) {
+ checkPartTypeNonNull(partType);
+ if (partType == PartType.COMPLETE)
+ return createToken(id, length);
+
+ if (isSkipToken(id)) {
+ operation.tokenRecognized(length, true);
+ return skipToken();
+ } else { // Do not skip the token
+ if (operation.tokenRecognized(length, false)) { // Create preprocessed token
+ return new ComplexToken(id, operation.tokenLength(), null, null, partType);
+ } else {
+ return new PropertyToken(id, operation.tokenLength(), null, partType);
+ }
+ }
+ }
+
+ /**
* Get flyweight token for the given arguments.
*
* Note: The returned token will not be flyweight under certain
@@ -165,9 +199,7 @@
* @param length >=0 length of the token to be created. The length must not
* exceed the number of characters read from the lexer input.
* @param propertyProvider non-null token property provider.
- * @param tokenStoreValue explicit value for property stored in the token itself
- * or null if there is no explicit value (value will be computed lazily when asked).
- *
+ * @param partType whether this token is complete or just a part of complete token.
* See {@link TokenPropertyProvider} for examples how this parameter may be used.
* @return non-null property token instance.
*
@@ -179,17 +211,18 @@
* because of token id filter.
*/
public Token createPropertyToken(T id, int length,
- TokenPropertyProvider propertyProvider, Object tokenStoreValue) {
+ TokenPropertyProvider propertyProvider, PartType partType) {
+ checkPartTypeNonNull(partType);
if (isSkipToken(id)) {
operation.tokenRecognized(length, true);
return skipToken();
} else { // Do not skip the token
if (operation.tokenRecognized(length, false)) { // Create preprocessed token
- return new PropertyPreprocessedTextToken(id, operation.tokenLength(),
- propertyProvider, tokenStoreValue);
+ return new ComplexToken(id, operation.tokenLength(),
+ propertyProvider, null, partType);
} else {
return new PropertyToken(id, operation.tokenLength(),
- propertyProvider, tokenStoreValue);
+ propertyProvider, partType);
}
}
}
@@ -198,16 +231,16 @@
* Create token with a custom text that possibly differs from the text
* represented by the token in the input text.
*/
- public Token createCustomTextToken(T id, CharSequence text, int length) {
+ public Token createCustomTextToken(T id, CharSequence text, int length, PartType partType) {
+ checkPartTypeNonNull(partType);
if (isSkipToken(id)) {
operation.tokenRecognized(length, true);
return skipToken();
} else { // Do not skip the token
if (operation.tokenRecognized(length, false)) { // Create preprocessed token
- return new PropertyCustomTextToken(id,
- operation.tokenLength(), text, null, null);
+ return new ComplexToken(id, operation.tokenLength(), null, text, partType);
} else {
- return new CustomTextToken(id, operation.tokenLength(), text);
+ return new CustomTextToken(id, operation.tokenLength(), text, partType);
}
}
}
@@ -220,6 +253,11 @@
@SuppressWarnings("unchecked") // NOI18N
private Token skipToken() {
return SKIP_TOKEN;
+ }
+
+ private void checkPartTypeNonNull(PartType partType) {
+ if (partType == null)
+ throw new IllegalArgumentException("partType must be non-null");
}
}
Index: lexer/src/org/netbeans/spi/lexer/TokenPropertyProvider.java
===================================================================
RCS file: /cvs/lexer/src/org/netbeans/spi/lexer/TokenPropertyProvider.java,v
retrieving revision 1.5
diff -u -r1.5 TokenPropertyProvider.java
--- lexer/src/org/netbeans/spi/lexer/TokenPropertyProvider.java 2 Feb 2007 14:47:35 -0000 1.5
+++ lexer/src/org/netbeans/spi/lexer/TokenPropertyProvider.java 3 Mar 2007 19:20:46 -0000
@@ -25,121 +25,53 @@
/**
* Provides extra properties of a token.
*
- * A special kind of token PropertyToken allows to carry token properties.
+ * Normally each token has an extra instance of the property provider:
+ *
+ * final class MyTokenPropertyProvider implements TokenPropertyProvider {
+ *
+ * private final Object value;
+ *
+ * TokenPropProvider(Object value) {
+ * this.value = value;
+ * }
+ *
+ * public Object getValue (Token token, Object key) {
+ * if ("type".equals(key))
+ * return value;
+ * return null;
+ * }
+ *
+ * }
+ *
*
- * That token may store a value of one property in its instance
- * (see PropertyToken.tokenStoreValue. If the provider
- * wants to use that field for storing of the value it needs to return
- * the corresponding key for that value from {@link #tokenStoreKey()}.
- *
- *
- * Generally this interface can be used in multiple ways:
- *
- * -
- * A new instance of the provider per each token.
- *
- * Example of storing a value for "key" key in the provider:
- *
- * final class MyTokenPropertyProvider implements TokenPropertyProvider {
- *
- * private final Object value;
- *
- * MyTokenPropertyProvider(Object value) {
- * this.value = value;
- * }
- *
- * public Object getValue(Token token, Object key) {
- * if ("key".equals(key)) {
- * return value;
- * }
- * return null;
- * }
- *
- * public Object getValue(Token token, Object tokenStoreKey, Object tokenStoreValue) {
- * return null; // not using token's storage
- * }
- *
- * public Object tokenStoreKey() {
- * return null; // not using token's storage
- * }
- *
- * }
- *
- *
- *
- * -
- * A single instance of the provider for multiple tokens.
- * Each token may have a specific value of the given property.
- *
- * This might be achieved by returning the particular key
- * from {@link #tokenStoreKey()} and using the token store value
- * for the storage of the property value.
- *
- * Example of storing a value for "key" by the token's storage:
- *
- * final class MyTokenPropertyProvider implements TokenPropertyProvider {
- *
- * static final MyTokenPropertyProvider INSTANCE = new MyTokenPropertyProvider();
- *
- * private MyTokenPropertyProvider() {
- * }
+ * However multiple flyweight instances of the provider may be used to save memory
+ * if there are just several values for a property.
+ *
+ * Example of two instances of a provider for boolean property "key":
+ *
+ * final class MyTokenPropertyProvider implements TokenPropertyProvider {
*
- * public Object getValue(Token token, Object key) {
- * return null; // no extra properties
- * }
- *
- * public Object getValue(Token token, Object tokenStoreKey, Object tokenStoreValue) {
- * // Assuming that the appropriate tokenStoreValue was passed to
- * // TokenFactory.createPropertyToken(id, length, propertyProvider, tokenStoreValue);
- * return tokenStoreValue;
- * }
- *
- * public Object tokenStoreKey() {
- * return "key";
- * }
- *
- * }
- *
- *
- *
- * -
- * Multiple flyweight instances of the provider.
- * This might be useful if there is just several values for the property.
- *
- * Example of two instances of a provider for "key":
- *
- * final class MyTokenPropertyProvider implements TokenPropertyProvider {
- *
- * static final MyTokenPropertyProvider TRUE = new MyTokenPropertyProvider(Boolean.TRUE);
- *
- * static final MyTokenPropertyProvider FALSE = new MyTokenPropertyProvider(Boolean.FALSE);
- *
- * private final Boolean value;
- *
- * private MyTokenPropertyProvider(Boolean value) {
- * this.value = value;
- * }
- *
- * public Object getValue(Token token, Object key) {
- * if ("key".equals(key)) {
- * return value;
- * }
- * return null;
- * }
- *
- * public Object getValue(Token token, Object tokenStoreKey, Object tokenStoreValue) {
- * return null; // not using token's storage
- * }
- *
- * public Object tokenStoreKey() {
- * return null; // not using token's storage
- * }
+ * static final MyTokenPropertyProvider TRUE = new MyTokenPropertyProvider(Boolean.TRUE);
+ *
+ * static final MyTokenPropertyProvider FALSE = new MyTokenPropertyProvider(Boolean.FALSE);
*
- * }
- *
- *
- *
+ * private final Boolean value;
+ *
+ * private MyTokenPropertyProvider(Boolean value) {
+ * this.value = value;
+ * }
+ *
+ * public Object getValue(Token token, Object key) {
+ * if ("key".equals(key)) {
+ * return value;
+ * }
+ * return null;
+ * }
*
+ * }
+ *
+ *
+ * A special kind of token PropertyToken allows to carry token properties.
*
* @author Miloslav Metelka
* @version 1.00
@@ -148,41 +80,13 @@
public interface TokenPropertyProvider {
/**
- * Get value of a property which is not a token-store property.
+ * Get value of a token property.
*
* @param token non-null token for which the property is being retrieved.
+ * It might be useful if the property would be computed dynamically.
* @param key non-null key for which the value should be retrieved.
* @return value of the property or null if there is no value for the given key.
*/
Object getValue(Token token, Object key);
-
- /**
- * Get value of a token-store property.
- *
- * This method is only invoked if {@link #tokenStoreKey()} returned non-null value.
- *
- * When called for the first time the tokenStoreValue
- * will have the value given to
- * {@link TokenFactory#createPropertyToken(TokenId,int,TokenPropertyProvider,Object)}.
- *
- * For subsequent invocations of this method the value returned from
- * a last call to it (for the given token) will be used.
- *
- * @param token non-null token for which the property is being retrieved.
- * @param tokenStoreKey non-null key for which the value should be retrieved.
- * @param tokenStoreValue value that was is currently stored in the token.
- * @return value for the tokenStoreKey. The value will be both returned
- * and stored in the token.
- */
- Object getValue(Token token, Object tokenStoreKey, Object tokenStoreValue);
-
- /**
- * Get a key of the property that is stored in the token.
- *
- * @return key of the property which is stored in the token itself
- * or null if the property provider does not want any property
- * to be stored in the token.
- */
- Object tokenStoreKey();
}
Index: lexer/test/unit/src/org/netbeans/api/lexer/CustomTokenClassTest.java
===================================================================
RCS file: /cvs/lexer/test/unit/src/org/netbeans/api/lexer/CustomTokenClassTest.java,v
retrieving revision 1.3
diff -u -r1.3 CustomTokenClassTest.java
--- lexer/test/unit/src/org/netbeans/api/lexer/CustomTokenClassTest.java 2 Feb 2007 14:47:37 -0000 1.3
+++ lexer/test/unit/src/org/netbeans/api/lexer/CustomTokenClassTest.java 3 Mar 2007 19:20:46 -0000
@@ -90,6 +90,10 @@
return false;
}
+ public PartType partType() {
+ return null;
+ }
+
}
}
Index: lexer/test/unit/src/org/netbeans/lib/lexer/test/LexerTestUtilities.java
===================================================================
RCS file: /cvs/lexer/test/unit/src/org/netbeans/lib/lexer/test/LexerTestUtilities.java,v
retrieving revision 1.8
diff -u -r1.8 LexerTestUtilities.java
--- lexer/test/unit/src/org/netbeans/lib/lexer/test/LexerTestUtilities.java 2 Feb 2007 14:47:43 -0000 1.8
+++ lexer/test/unit/src/org/netbeans/lib/lexer/test/LexerTestUtilities.java 3 Mar 2007 19:20:47 -0000
@@ -217,6 +217,7 @@
TestCase.assertEquals(message + "Invalid token.state()", state(ts), state(ts2));
}
TestCase.assertEquals(message + "Invalid token length", t.length(), t2.length());
+ TestCase.assertEquals(message + "Invalid token part", t.partType(), t2.partType());
}
/**
Index: lexer/test/unit/src/org/netbeans/lib/lexer/test/dump/TokenDumpCheck.java
===================================================================
RCS file: /cvs/lexer/test/unit/src/org/netbeans/lib/lexer/test/dump/TokenDumpCheck.java,v
retrieving revision 1.3
diff -u -r1.3 TokenDumpCheck.java
--- lexer/test/unit/src/org/netbeans/lib/lexer/test/dump/TokenDumpCheck.java 2 Feb 2007 14:47:46 -0000 1.3
+++ lexer/test/unit/src/org/netbeans/lib/lexer/test/dump/TokenDumpCheck.java 3 Mar 2007 19:20:47 -0000
@@ -27,7 +27,6 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import java.util.regex.Pattern;
import org.netbeans.api.lexer.Language;
import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenHierarchy;
Index: lexer/test/unit/src/org/netbeans/lib/lexer/test/dump/TokenDumpLexer.java
===================================================================
RCS file: /cvs/lexer/test/unit/src/org/netbeans/lib/lexer/test/dump/TokenDumpLexer.java,v
retrieving revision 1.3
diff -u -r1.3 TokenDumpLexer.java
--- lexer/test/unit/src/org/netbeans/lib/lexer/test/dump/TokenDumpLexer.java 2 Feb 2007 14:47:46 -0000 1.3
+++ lexer/test/unit/src/org/netbeans/lib/lexer/test/dump/TokenDumpLexer.java 3 Mar 2007 19:20:47 -0000
@@ -19,6 +19,7 @@
package org.netbeans.lib.lexer.test.dump;
+import org.netbeans.api.lexer.PartType;
import org.netbeans.api.lexer.Token;
import org.netbeans.spi.lexer.Lexer;
import org.netbeans.spi.lexer.LexerInput;
@@ -182,7 +183,7 @@
case EOF:
input.backup(1);
return tokenFactory.createPropertyToken(id, input.readLength(),
- UnicodeCharValueProvider.INSTANCE, new Character(ch));
+ new UnicodeCharValueProvider(new Character(ch)), PartType.COMPLETE);
}
}
input.backup(1);
@@ -198,18 +199,16 @@
private static final class UnicodeCharValueProvider implements TokenPropertyProvider {
- static final UnicodeCharValueProvider INSTANCE = new UnicodeCharValueProvider();
+ private Character ch;
- public Object getValue(Token token, Object key) {
- return null; // no non-tokenStore value
- }
-
- public Object getValue(Token token, Object tokenStoreKey, Object tokenStoreValue) {
- return tokenStoreValue;
+ UnicodeCharValueProvider(Character ch) {
+ this.ch = ch;
}
- public Object tokenStoreKey() {
- return TokenDumpTokenId.UNICODE_CHAR_TOKEN_PROPERTY;
+ public Object getValue(Token token, Object key) {
+ if (TokenDumpTokenId.UNICODE_CHAR_TOKEN_PROPERTY.equals(key))
+ return ch;
+ return null; // no non-tokenStore value
}
}
Index: web/jspsyntax/lexer/nbproject/project.xml
===================================================================
RCS file: /cvs/web/jspsyntax/lexer/nbproject/project.xml,v
retrieving revision 1.7
diff -u -r1.7 project.xml
--- web/jspsyntax/lexer/nbproject/project.xml 12 Jan 2007 13:15:58 -0000 1.7
+++ web/jspsyntax/lexer/nbproject/project.xml 3 Mar 2007 19:21:05 -0000
@@ -46,7 +46,7 @@
1
- 1.1
+ 1.2