| StringTokenizer.java |
1 /*
2 * %W% %E%
3 *
4 * Copyright (c) 2006, Oracle and/or its affiliates. All rights reserved.
5 * ORACLE PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
6 */
7
8 package java.util;
9
10 import java.lang.*;
11
12 /**
13 * The string tokenizer class allows an application to break a
14 * string into tokens. The tokenization method is much simpler than
15 * the one used by the <code>StreamTokenizer</code> class. The
16 * <code>StringTokenizer</code> methods do not distinguish among
17 * identifiers, numbers, and quoted strings, nor do they recognize
18 * and skip comments.
19 * <p>
20 * The set of delimiters (the characters that separate tokens) may
21 * be specified either at creation time or on a per-token basis.
22 * <p>
23 * An instance of <code>StringTokenizer</code> behaves in one of two
24 * ways, depending on whether it was created with the
25 * <code>returnDelims</code> flag having the value <code>true</code>
26 * or <code>false</code>:
27 * <ul>
28 * <li>If the flag is <code>false</code>, delimiter characters serve to
29 * separate tokens. A token is a maximal sequence of consecutive
30 * characters that are not delimiters.
31 * <li>If the flag is <code>true</code>, delimiter characters are themselves
32 * considered to be tokens. A token is thus either one delimiter
33 * character, or a maximal sequence of consecutive characters that are
34 * not delimiters.
35 * </ul><p>
36 * A <tt>StringTokenizer</tt> object internally maintains a current
37 * position within the string to be tokenized. Some operations advance this
38 * current position past the characters processed.<p>
39 * A token is returned by taking a substring of the string that was used to
40 * create the <tt>StringTokenizer</tt> object.
41 * <p>
42 * The following is one example of the use of the tokenizer. The code:
43 * <blockquote><pre>
44 * StringTokenizer st = new StringTokenizer("this is a test");
45 * while (st.hasMoreTokens()) {
46 * System.out.println(st.nextToken());
47 * }
48 * </pre></blockquote>
49 * <p>
50 * prints the following output:
51 * <blockquote><pre>
52 * this
53 * is
54 * a
55 * test
56 * </pre></blockquote>
57 *
58 * <p>
59 * <tt>StringTokenizer</tt> is a legacy class that is retained for
60 * compatibility reasons although its use is discouraged in new code. It is
61 * recommended that anyone seeking this functionality use the <tt>split</tt>
62 * method of <tt>String</tt> or the java.util.regex package instead.
63 * <p>
64 * The following example illustrates how the <tt>String.split</tt>
65 * method can be used to break up a string into its basic tokens:
66 * <blockquote><pre>
67 * String[] result = "this is a test".split("\\s");
68 * for (int x=0; x<result.length; x++)
69 * System.out.println(result[x]);
70 * </pre></blockquote>
71 * <p>
72 * prints the following output:
73 * <blockquote><pre>
74 * this
75 * is
76 * a
77 * test
78 * </pre></blockquote>
79 *
80 * @author unascribed
81 * @version %I%, %G%
82 * @see java.io.StreamTokenizer
83 * @since JDK1.0
84 */
85 public
86 class StringTokenizer implements Enumeration<Object> {
87 private int currentPosition;
88 private int newPosition;
89 private int maxPosition;
90 private String str;
91 private String delimiters;
92 private boolean retDelims;
93 private boolean delimsChanged;
94
95 /**
96 * maxDelimCodePoint stores the value of the delimiter character with the
97 * highest value. It is used to optimize the detection of delimiter
98 * characters.
99 *
100 * It is unlikely to provide any optimization benefit in the
101 * hasSurrogates case because most string characters will be
102 * smaller than the limit, but we keep it so that the two code
103 * paths remain similar.
104 */
105 private int maxDelimCodePoint;
106
107 /**
108 * If delimiters include any surrogates (including surrogate
109 * pairs), hasSurrogates is true and the tokenizer uses the
110 * different code path. This is because String.indexOf(int)
111 * doesn't handle unpaired surrogates as a single character.
112 */
113 private boolean hasSurrogates = false;
114
115 /**
116 * When hasSurrogates is true, delimiters are converted to code
117 * points and isDelimiter(int) is used to determine if the given
118 * codepoint is a delimiter.
119 */
120 private int[] delimiterCodePoints;
121
122 /**
123 * Set maxDelimCodePoint to the highest char in the delimiter set.
124 */
125 private void setMaxDelimCodePoint() {
126 if (delimiters == null) {
127 maxDelimCodePoint = 0;
128 return;
129 }
130
131 int m = 0;
132 int c;
133 int count = 0;
134 for (int i = 0; i < delimiters.length(); i += Character.charCount(c)) {
135 c = delimiters.charAt(i);
136 if (c >= Character.MIN_HIGH_SURROGATE && c <= Character.MAX_LOW_SURROGATE) {
137 c = delimiters.codePointAt(i);
138 hasSurrogates = true;
139 }
140 if (m < c)
141 m = c;
142 count++;
143 }
144 maxDelimCodePoint = m;
145
146 if (hasSurrogates) {
147 delimiterCodePoints = new int[count];
148 for (int i = 0, j = 0; i < count; i++, j += Character.charCount(c)) {
149 c = delimiters.codePointAt(j);
150 delimiterCodePoints[i] = c;
151 }
152 }
153 }
154
155 /**
156 * Constructs a string tokenizer for the specified string. All
157 * characters in the <code>delim</code> argument are the delimiters
158 * for separating tokens.
159 * <p>
160 * If the <code>returnDelims</code> flag is <code>true</code>, then
161 * the delimiter characters are also returned as tokens. Each
162 * delimiter is returned as a string of length one. If the flag is
163 * <code>false</code>, the delimiter characters are skipped and only
164 * serve as separators between tokens.
165 * <p>
166 * Note that if <tt>delim</tt> is <tt>null</tt>, this constructor does
167 * not throw an exception. However, trying to invoke other methods on the
168 * resulting <tt>StringTokenizer</tt> may result in a
169 * <tt>NullPointerException</tt>.
170 *
171 * @param str a string to be parsed.
172 * @param delim the delimiters.
173 * @param returnDelims flag indicating whether to return the delimiters
174 * as tokens.
175 * @exception NullPointerException if str is <CODE>null</CODE>
176 */
177 public StringTokenizer(String str, String delim, boolean returnDelims) {
178 currentPosition = 0;
179 newPosition = -1;
180 delimsChanged = false;
181 this.str = str;
182 maxPosition = str.length();
183 delimiters = delim;
184 retDelims = returnDelims;
185 setMaxDelimCodePoint();
186 }
187
188 /**
189 * Constructs a string tokenizer for the specified string. The
190 * characters in the <code>delim</code> argument are the delimiters
191 * for separating tokens. Delimiter characters themselves will not
192 * be treated as tokens.
193 * <p>
194 * Note that if <tt>delim</tt> is <tt>null</tt>, this constructor does
195 * not throw an exception. However, trying to invoke other methods on the
196 * resulting <tt>StringTokenizer</tt> may result in a
197 * <tt>NullPointerException</tt>.
198 *
199 * @param str a string to be parsed.
200 * @param delim the delimiters.
201 * @exception NullPointerException if str is <CODE>null</CODE>
202 */
203 public StringTokenizer(String str, String delim) {
204 this(str, delim, false);
205 }
206
207 /**
208 * Constructs a string tokenizer for the specified string. The
209 * tokenizer uses the default delimiter set, which is
210 * <code>" \t\n\r\f"</code>: the space character,
211 * the tab character, the newline character, the carriage-return character,
212 * and the form-feed character. Delimiter characters themselves will
213 * not be treated as tokens.
214 *
215 * @param str a string to be parsed.
216 * @exception NullPointerException if str is <CODE>null</CODE>
217 */
218 public StringTokenizer(String str) {
219 this(str, " \t\n\r\f", false);
220 }
221
222 /**
223 * Skips delimiters starting from the specified position. If retDelims
224 * is false, returns the index of the first non-delimiter character at or
225 * after startPos. If retDelims is true, startPos is returned.
226 */
227 private int skipDelimiters(int startPos) {
228 if (delimiters == null)
229 throw new NullPointerException();
230
231 int position = startPos;
232 while (!retDelims && position < maxPosition) {
233 if (!hasSurrogates) {
234 char c = str.charAt(position);
235 if ((c > maxDelimCodePoint) || (delimiters.indexOf(c) < 0))
236 break;
237 position++;
238 } else {
239 int c = str.codePointAt(position);
240 if ((c > maxDelimCodePoint) || !isDelimiter(c)) {
241 break;
242 }
243 position += Character.charCount(c);
244 }
245 }
246 return position;
247 }
248
249 /**
250 * Skips ahead from startPos and returns the index of the next delimiter
251 * character encountered, or maxPosition if no such delimiter is found.
252 */
253 private int scanToken(int startPos) {
254 int position = startPos;
255 while (position < maxPosition) {
256 if (!hasSurrogates) {
257 char c = str.charAt(position);
258 if ((c <= maxDelimCodePoint) && (delimiters.indexOf(c) >= 0))
259 break;
260 position++;
261 } else {
262 int c = str.codePointAt(position);
263 if ((c <= maxDelimCodePoint) && isDelimiter(c))
264 break;
265 position += Character.charCount(c);
266 }
267 }
268 if (retDelims && (startPos == position)) {
269 if (!hasSurrogates) {
270 char c = str.charAt(position);
271 if ((c <= maxDelimCodePoint) && (delimiters.indexOf(c) >= 0))
272 position++;
273 } else {
274 int c = str.codePointAt(position);
275 if ((c <= maxDelimCodePoint) && isDelimiter(c))
276 position += Character.charCount(c);
277 }
278 }
279 return position;
280 }
281
282 private boolean isDelimiter(int codePoint) {
283 for (int i = 0; i < delimiterCodePoints.length; i++) {
284 if (delimiterCodePoints[i] == codePoint) {
285 return true;
286 }
287 }
288 return false;
289 }
290
291 /**
292 * Tests if there are more tokens available from this tokenizer's string.
293 * If this method returns <tt>true</tt>, then a subsequent call to
294 * <tt>nextToken</tt> with no argument will successfully return a token.
295 *
296 * @return <code>true</code> if and only if there is at least one token
297 * in the string after the current position; <code>false</code>
298 * otherwise.
299 */
300 public boolean hasMoreTokens() {
301 /*
302 * Temporarily store this position and use it in the following
303 * nextToken() method only if the delimiters haven't been changed in
304 * that nextToken() invocation.
305 */
306 newPosition = skipDelimiters(currentPosition);
307 return (newPosition < maxPosition);
308 }
309
310 /**
311 * Returns the next token from this string tokenizer.
312 *
313 * @return the next token from this string tokenizer.
314 * @exception NoSuchElementException if there are no more tokens in this
315 * tokenizer's string.
316 */
317 public String nextToken() {
318 /*
319 * If next position already computed in hasMoreElements() and
320 * delimiters have changed between the computation and this invocation,
321 * then use the computed value.
322 */
323
324 currentPosition = (newPosition >= 0 && !delimsChanged) ?
325 newPosition : skipDelimiters(currentPosition);
326
327 /* Reset these anyway */
328 delimsChanged = false;
329 newPosition = -1;
330
331 if (currentPosition >= maxPosition)
332 throw new NoSuchElementException();
333 int start = currentPosition;
334 currentPosition = scanToken(currentPosition);
335 return str.substring(start, currentPosition);
336 }
337
338 /**
339 * Returns the next token in this string tokenizer's string. First,
340 * the set of characters considered to be delimiters by this
341 * <tt>StringTokenizer</tt> object is changed to be the characters in
342 * the string <tt>delim</tt>. Then the next token in the string
343 * after the current position is returned. The current position is
344 * advanced beyond the recognized token. The new delimiter set
345 * remains the default after this call.
346 *
347 * @param delim the new delimiters.
348 * @return the next token, after switching to the new delimiter set.
349 * @exception NoSuchElementException if there are no more tokens in this
350 * tokenizer's string.
351 * @exception NullPointerException if delim is <CODE>null</CODE>
352 */
353 public String nextToken(String delim) {
354 delimiters = delim;
355
356 /* delimiter string specified, so set the appropriate flag. */
357 delimsChanged = true;
358
359 setMaxDelimCodePoint();
360 return nextToken();
361 }
362
363 /**
364 * Returns the same value as the <code>hasMoreTokens</code>
365 * method. It exists so that this class can implement the
366 * <code>Enumeration</code> interface.
367 *
368 * @return <code>true</code> if there are more tokens;
369 * <code>false</code> otherwise.
370 * @see java.util.Enumeration
371 * @see java.util.StringTokenizer#hasMoreTokens()
372 */
373 public boolean hasMoreElements() {
374 return hasMoreTokens();
375 }
376
377 /**
378 * Returns the same value as the <code>nextToken</code> method,
379 * except that its declared return value is <code>Object</code> rather than
380 * <code>String</code>. It exists so that this class can implement the
381 * <code>Enumeration</code> interface.
382 *
383 * @return the next token in the string.
384 * @exception NoSuchElementException if there are no more tokens in this
385 * tokenizer's string.
386 * @see java.util.Enumeration
387 * @see java.util.StringTokenizer#nextToken()
388 */
389 public Object nextElement() {
390 return nextToken();
391 }
392
393 /**
394 * Calculates the number of times that this tokenizer's
395 * <code>nextToken</code> method can be called before it generates an
396 * exception. The current position is not advanced.
397 *
398 * @return the number of tokens remaining in the string using the current
399 * delimiter set.
400 * @see java.util.StringTokenizer#nextToken()
401 */
402 public int countTokens() {
403 int count = 0;
404 int currpos = currentPosition;
405 while (currpos < maxPosition) {
406 currpos = skipDelimiters(currpos);
407 if (currpos >= maxPosition)
408 break;
409 currpos = scanToken(currpos);
410 count++;
411 }
412 return count;
413 }
414 }
415