Edinburgh Speech Tools  2.4-release
EST_Token.cc
1 /*************************************************************************/
2 /* */
3 /* Centre for Speech Technology Research */
4 /* University of Edinburgh, UK */
5 /* Copyright (c) 1996 */
6 /* All Rights Reserved. */
7 /* */
8 /* Permission is hereby granted, free of charge, to use and distribute */
9 /* this software and its documentation without restriction, including */
10 /* without limitation the rights to use, copy, modify, merge, publish, */
11 /* distribute, sublicense, and/or sell copies of this work, and to */
12 /* permit persons to whom this work is furnished to do so, subject to */
13 /* the following conditions: */
14 /* 1. The code must retain the above copyright notice, this list of */
15 /* conditions and the following disclaimer. */
16 /* 2. Any modifications must be clearly marked as such. */
17 /* 3. Original authors' names are not deleted. */
18 /* 4. The authors' names are not used to endorse or promote products */
19 /* derived from this software without specific prior written */
20 /* permission. */
21 /* */
22 /* THE UNIVERSITY OF EDINBURGH AND THE CONTRIBUTORS TO THIS WORK */
23 /* DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING */
24 /* ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT */
25 /* SHALL THE UNIVERSITY OF EDINBURGH NOR THE CONTRIBUTORS BE LIABLE */
26 /* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES */
27 /* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN */
28 /* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, */
29 /* ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF */
30 /* THIS SOFTWARE. */
31 /* */
32 /*************************************************************************/
33 /* Author : Alan W Black */
34 /* Date : April 1996 */
35 /*-----------------------------------------------------------------------*/
36 /* */
37 /* A Tokenize class, both for Tokens (Strings plus alpha) */
38 /* EST_TokenStream for strings, FILE *, files, pipes etc */
39 /* */
40 /*=======================================================================*/
41 #include <cstdio>
42 #include <iostream>
43 #include "EST_unix.h"
44 #include <cstdlib>
45 #include <climits>
46 #include <cstring>
47 #include "EST_math.h"
48 #include "EST_Token.h"
49 #include "EST_string_aux.h"
50 #include "EST_cutils.h"
51 #include "EST_error.h"
52 
53 const EST_String EST_Token_Default_WhiteSpaceChars = " \t\n\r";
54 const EST_String EST_Token_Default_SingleCharSymbols = "(){}[]";
55 const EST_String EST_Token_Default_PrePunctuationSymbols = "\"'`({[";
56 const EST_String EST_Token_Default_PunctuationSymbols = "\"'`.,:;!?]})";
57 const EST_String Token_Origin_FD = "existing file descriptor";
58 const EST_String Token_Origin_Stream = "existing istream";
59 const EST_String Token_Origin_String = "existing string";
60 
61 static EST_Regex RXanywhitespace("[ \t\n\r]");
62 
63 static inline char *check_extend_str_in(char *str, int pos, int *max)
64 {
65  // Check we are not at the end of the string, if so get some more
66  // and copy the old one into the new one
67  char *newstuff;
68 
69  if (pos >= *max)
70  {
71  if (pos > *max)
72  *max = 2 * pos;
73  else
74  *max *= 2;
75  newstuff = new char[*max];
76  strncpy(newstuff,str,pos);
77  delete [] str;
78  return newstuff;
79  }
80  else
81  return str;
82 }
83 
84 #define check_extend_str(STR, POS, MAX) \
85  (((POS)>= *(MAX))?check_extend_str_in((STR),(POS),(MAX)):(STR))
86 
87 ostream& operator<<(ostream& s, const EST_Token &p)
88 {
89  s << "[TOKEN " << p.pname << "]";
90  return s;
91 }
92 
93 
94 EST_Token &EST_Token::operator = (const EST_Token &a)
95 {
96  linenum = a.linenum;
97  linepos = a.linepos;
98  p_filepos = a.p_filepos;
99  p_quoted = a.p_quoted;
100  space = a.space;
101  prepunc = a.prepunc;
102  pname = a.pname;
103  punc = a.punc;
104  return *this;
105 }
106 
108 {
109  return "line "+itoString(linenum)+" char "+itoString(linepos);
110 }
111 
112 EST_Token &EST_Token::operator = (const EST_String &a)
113 {
114  pname = a;
115  return *this;
116 }
117 
118 EST_TokenStream::EST_TokenStream()
119 {
120  tok_wspacelen = 64; // will grow if necessary
121  tok_wspace = new char[tok_wspacelen];
122  tok_stufflen = 512; // will grow if necessary
123  tok_stuff = new char[tok_stufflen];
124  tok_prepuncslen = 32; // will grow if necessary
125  tok_prepuncs = new char[tok_prepuncslen];
126 
127  default_values();
128 }
129 
130 EST_TokenStream::EST_TokenStream(EST_TokenStream &s)
131 {
132  (void)s;
133 
134  cerr << "TokenStream: warning passing TokenStream not as reference"
135  << endl;
136 
137  // You *really* shouldn't use this AT ALL unless you
138  // fully understand its consequences, you'll be copying open
139  // files and moving file pointers all over the place
140  // basically *DON'T* do this, pass the stream by reference
141 
142  // Now there may be occasions when you do want to do this for example
143  // when you need to do far look ahead or check point as you read
144  // but they are obscure and I'm not sure how to do that for all
145  // the file forms supported by the TokenStream. If you do
146  // I can write a clone function that might do it.
147 
148 }
149 
150 void EST_TokenStream::default_values()
151 {
152  type = tst_none;
153  peeked_tokp = FALSE;
154  peeked_charp = FALSE;
155  eof_flag = FALSE;
156  quotes = FALSE;
157  p_filepos = 0;
158  linepos = 1;
159  WhiteSpaceChars = EST_Token_Default_WhiteSpaceChars;
160  SingleCharSymbols = EST_String::Empty;
161  PrePunctuationSymbols = EST_String::Empty;
162  PunctuationSymbols = EST_String::Empty;
163  build_table();
164  close_at_end=TRUE;
165 }
166 
168 {
169  if (type != tst_none)
170  close();
171  delete [] tok_wspace;
172  delete [] tok_stuff;
173  delete [] tok_prepuncs;
174 
175 }
176 
177 ostream& operator<<(ostream& s, EST_TokenStream &p)
178 {
179  s << "[TOKENSTREAM ";
180  switch (p.type)
181  {
182  case tst_none:
183  cerr << "UNSET"; break;
184  case tst_file:
185  cerr << "FILE"; break;
186  case tst_pipe:
187  cerr << "PIPE"; break;
188  case tst_istream:
189  cerr << "ISTREAM"; break;
190  case tst_string:
191  cerr << "STRING"; break;
192  default:
193  cerr << "UNKNOWN" << endl;
194  }
195  s << "]";
196 
197  return s;
198 }
199 
200 int EST_TokenStream::open(const EST_String &filename)
201 {
202  if (type != tst_none)
203  close();
204  default_values();
205  fp = fopen(filename,"rb");
206  if (fp == NULL)
207  {
208  cerr << "Cannot open file " << filename << " as tokenstream"
209  << endl;
210  return -1;
211  }
212  Origin = filename;
213  type = tst_file;
214 
215  return 0;
216 }
217 
218 int EST_TokenStream::open(FILE *ofp, int close_when_finished)
219 {
220  // absorb already open stream
221  if (type != tst_none)
222  close();
223  default_values();
224  fp = ofp;
225  if (fp == NULL)
226  {
227  cerr << "Cannot absorb NULL filestream as tokenstream" << endl;
228  return -1;
229  }
230  Origin = Token_Origin_FD;
231  type = tst_file;
232 
233  close_at_end = close_when_finished;
234 
235  return 0;
236 }
237 
238 int EST_TokenStream::open(istream &newis)
239 {
240  // absorb already open istream
241  if (type != tst_none)
242  close();
243  default_values();
244  is = &newis;
245  Origin = Token_Origin_Stream;
246  type = tst_istream;
247 
248  return 0;
249 }
250 
252 {
253  // Make a tokenstream from an internal existing string/buffer
254  const char *buf;
255  if (type != tst_none)
256  close();
257  default_values();
258  buf = (const char *)newbuffer;
259  buffer_length = newbuffer.length();
260  buffer = new char[buffer_length+1];
261  memmove(buffer,buf,buffer_length+1);
262  pos = 0;
263  Origin = Token_Origin_String;
264  type = tst_string;
265 
266  return 0;
267 }
268 
269 int EST_TokenStream::seek_end()
270 {
271  // This isn't actually useful but people expect it
272  peeked_charp = FALSE;
273  peeked_tokp = FALSE;
274 
275  switch (type)
276  {
277  case tst_none:
278  cerr << "EST_TokenStream unset" << endl;
279  return -1;
280  break;
281  case tst_file:
282  fseek(fp,0,SEEK_END);
283  p_filepos = ftell(fp);
284  return p_filepos;
285  case tst_pipe:
286  cerr << "EST_TokenStream seek on pipe not supported" << endl;
287  return -1;
288  break;
289  case tst_istream:
290  is->seekg(0,is->end);
291  p_filepos = is->tellg();
292  return p_filepos;
293  break;
294  case tst_string:
295  pos = buffer_length;
296  return pos;
297  default:
298  cerr << "EST_TokenStream: unknown type" << endl;
299  return -1;
300  }
301 
302  return -1; // can't get here
303 }
304 
305 int EST_TokenStream::seek(int position)
306 {
307  peeked_charp = FALSE;
308  peeked_tokp = FALSE;
309 
310  switch (type)
311  {
312  case tst_none:
313  cerr << "EST_TokenStream unset" << endl;
314  return -1;
315  break;
316  case tst_file:
317  p_filepos = position;
318  return fseek(fp,position,SEEK_SET);
319  case tst_pipe:
320  cerr << "EST_TokenStream seek on pipe not supported" << endl;
321  return -1;
322  break;
323  case tst_istream:
324  p_filepos = position;
325  is->seekg(position, is->beg);
326  return 0;
327  break;
328  case tst_string:
329  if (position >= pos)
330  {
331  pos = position;
332  return -1;
333  }
334  else
335  {
336  pos = position;
337  return 0;
338  }
339  break;
340  default:
341  cerr << "EST_TokenStream: unknown type" << endl;
342  return -1;
343  }
344 
345  return -1; // can't get here
346 
347 }
348 
349 static int stdio_fread(void *buff,int size,int nitems,FILE *fp)
350 {
351  // So it can find the stdio one rather than the TokenStream one
352  return fread(buff,size,nitems,fp);
353 }
354 
355 int EST_TokenStream::fread(void *buff, int size, int nitems)
356 {
357  // switching into binary mode for current position
358  int items_read;
359 
360  // so we can continue to read afterwards
361  if (peeked_tokp)
362  {
363  cerr << "ERROR " << pos_description()
364  << " peeked into binary data" << endl;
365  return 0;
366  }
367 
368  peeked_charp = FALSE;
369  peeked_tokp = FALSE;
370 
371  switch (type)
372  {
373  case tst_none:
374  cerr << "EST_TokenStream unset" << endl;
375  return 0;
376  break;
377  case tst_file:
378  items_read = stdio_fread(buff,(size_t)size,(size_t)nitems,fp);
379  p_filepos += items_read*size;
380  return items_read;
381  case tst_pipe:
382  cerr << "EST_TokenStream fread pipe not yet supported" << endl;
383  return 0;
384  break;
385  case tst_istream:
386  is->read((char*)buff, (size_t) size*nitems);
387  return is->gcount()/size;
388  break;
389  case tst_string:
390  if ((buffer_length-pos)/size < nitems)
391  items_read = (buffer_length-pos)/size;
392  else
393  items_read = nitems;
394  memcpy(buff,&buffer[pos],items_read*size);
395  pos += items_read*size;
396  return items_read;
397  default:
398  cerr << "EST_TokenStream: unknown type" << endl;
399  return EOF;
400  }
401 
402  return 0; // can't get here
403 
404 }
405 
407 {
408  // close any files (if they were used)
409 
410  switch (type)
411  {
412  case tst_none:
413  break;
414  case tst_file:
415  if (close_at_end)
416  fclose(fp);
417  case tst_pipe:
418  // close(fd);
419  break;
420  case tst_istream:
421  break;
422  case tst_string:
423  delete [] buffer;
424  buffer = 0;
425  break;
426  default:
427  cerr << "EST_TokenStream: unknown type" << endl;
428  break;
429  }
430 
431  type = tst_none;
432  peeked_charp = FALSE;
433  peeked_tokp = FALSE;
434 
435 }
436 
438 {
439  // For paul, the only person I know who uses this
440 
441  switch (type)
442  {
443  case tst_none:
444  break;
445  case tst_file:
446  fp = freopen(Origin,"rb",fp);
447  p_filepos = 0;
448  break;
449  case tst_pipe:
450  cerr << "EST_TokenStream: can't rewind pipe" << endl;
451  return -1;
452  break;
453  case tst_istream:
454  cerr << "EST_TokenStream: can't rewind istream" << endl;
455  break;
456  case tst_string:
457  pos = 0;
458  break;
459  default:
460  cerr << "EST_TokenStream: unknown type" << endl;
461  break;
462  }
463 
464  linepos = 1;
465  peeked_charp = FALSE;
466  peeked_tokp = FALSE;
467  eof_flag = FALSE;
468 
469  return 0;
470 }
471 
472 EST_TokenStream & EST_TokenStream::operator >>(EST_Token &p)
473 {
474  return get(p);
475 }
476 
477 EST_TokenStream & EST_TokenStream::operator >>(EST_String &p)
478 {
479  EST_Token t;
480 
481  get(t);
482  p = t.string();
483  return *this;
484 }
485 
487 {
488  tok = get();
489  return *this;
490 }
491 
493 {
494  // Returns a concatenated token form here to next symbol that matches s
495  // including s (though not adding s on the result)
496  // Not really for the purist but lots of times very handy
497  // Note this is not very efficient
498  EST_String result;
499  EST_Token t;
500 
501  for (result=EST_String::Empty; (t=get()) != s; )
502  {
503  result += t.whitespace() + t.prepunctuation() +
504  t.string() + t.punctuation();
505  if (eof())
506  {
507  cerr << "EST_TokenStream: end of file when looking for \"" <<
508  s << "\"" << endl;
509  break;
510  }
511  }
512 
513  return EST_Token(result);
514 }
515 
517 {
518  // Swallow the lot up to end of line
519  // assumes \n is a whitespace character
520 
522 
523  while (!eoln())
524  {
525  EST_Token &t=get();
526  result += t.whitespace() + t.prepunctuation();
527 
528  if (quotes)
529  result += quote_string(t.string());
530  else
531  result += t.string();
532 
533  result += t.punctuation();
534 
535  if (eof())
536  {
537 // cerr << "EST_TokenStream: end of file when looking for end of line"
538 // << endl;
539  break;
540  }
541  }
542  // So that the next call works I have to step over the eoln condition
543  // That involves removing the whitespace upto and including the next
544  // \n in the peek token.
545 
546  char *w = wstrdup(peek().whitespace());
547  int i;
548  for (i=0; w[i] != 0; i++)
549  if (w[i] == '\n') // maybe not portable
550  peek().set_whitespace(&w[i+1]);
551 
552  wfree(w);
553 
554  static EST_Token result_t;
555 
556  result_t.set_token(result);
557 
558  return result_t;
559 }
560 
561 EST_Token &EST_TokenStream::must_get(EST_String expected, bool *ok)
562 {
563  EST_Token &tok = get();
564 
565  if (tok != expected)
566  {
567  if (ok != NULL)
568  {
569  *ok=FALSE;
570  return tok;
571  }
572  else
573  EST_error("Expected '%s' got '%s' at %s",
574  (const char *)expected,
575  (const char *)(EST_String)tok,
576  (const char *)pos_description());
577  }
578 
579  if (ok != NULL)
580  *ok=TRUE;
581  return tok;
582 }
583 
584 void EST_TokenStream::build_table()
585 {
586  int i;
587  const char *p;
588  unsigned char c;
589 
590  for (i=0; i<256; ++i)
591  p_table[i]=0;
592 
593  for (p=WhiteSpaceChars; *p; ++p)
594  if (p_table[c=(unsigned char)*p])
595  EST_warning("Character '%c' has two classes, '%c' and '%c'",
596  *p, c, ' ');
597  else
598  p_table[c] = ' ';
599 
600  for (p=SingleCharSymbols; *p; ++p)
601  if (p_table[c=(unsigned char)*p])
602  EST_warning("Character '%c' has two classes, '%c' and '%c'",
603  *p, p_table[c], '!');
604  else
605  p_table[c] = '@';
606 
607  for (p=PunctuationSymbols; *p; ++p)
608  if (p_table[c=(unsigned char)*p] == '@')
609  continue;
610  else if (p_table[c])
611  EST_warning("Character '%c' has two classes, '%c' and '%c'",
612  *p, p_table[c], '.');
613  else
614  p_table[c] = '.';
615 
616  for(p=PrePunctuationSymbols; *p; ++p)
617  if (p_table[c=(unsigned char)*p] == '@')
618  continue;
619  else if (p_table[c] == '.')
620  p_table[c] = '"';
621  else if (p_table[c])
622  EST_warning("Character '%c' has two classes, '%c' and '%c'",
623  *p, p_table[c], '$');
624  else
625  p_table[c] = '$';
626 
627  p_table_wrong=0;
628 }
629 
630 inline int EST_TokenStream::getpeeked_internal(void)
631 {
632  peeked_charp = FALSE;
633  return peeked_char;
634 }
635 
636 inline
637 int EST_TokenStream::getch_internal()
638 {
639  // Return next character in stream
640  if (EST_TokenStream::peeked_charp)
641  {
642  return getpeeked_internal();
643  }
644 
645  switch (type)
646  {
647  case tst_none:
648  cerr << "EST_TokenStream unset" << endl;
649  return EOF;
650  break;
651  case tst_file:
652  p_filepos++;
653  {
654  char lc;
655  if (stdio_fread(&lc,1,1,fp) == 0)
656  return EOF;
657  else
658  return (int)lc;
659  }
660 /* return getc(fp); */
661  case tst_pipe:
662  cerr << "EST_TokenStream pipe not yet supported" << endl;
663  return EOF;
664  break;
665  case tst_istream:
666  p_filepos++;
667  return is->get();
668  case tst_string:
669  if (pos < buffer_length)
670  {
671  p_filepos++;
672  return buffer[pos++];
673  }
674  else
675  return EOF;
676  default:
677  cerr << "EST_TokenStream: unknown type" << endl;
678  return EOF;
679  }
680 
681  return EOF; // can't get here
682 }
683 
684 int EST_TokenStream::getch(void)
685 {
686  return getch_internal();
687 }
688 
689 inline int EST_TokenStream::peekch_internal()
690 {
691  // Return next character in stream (without reading it)
692 
693  if (!peeked_charp)
694  peeked_char = getch_internal();
695  peeked_charp = TRUE;
696  return peeked_char;
697 }
698 
699 
700 int EST_TokenStream::peekch(void)
701 {
702  return peekch_internal();
703 
704 }
705 
706 #define CLASS(C,CL) (p_table[(unsigned char)(C)]==(CL))
707 
708 #define CLASS2(C,CL1,CL2) (p_table[(unsigned char)(C)]==(CL1)||p_table[(unsigned char)(C)]==(CL2))
709 
711 {
712  if (peeked_tokp)
713  {
714  peeked_tokp = FALSE;
715  return current_tok;
716  }
717 
718  if (p_table_wrong)
719  build_table();
720 
721  char *word;
722  int c,i,j;
723 
724  for (i=0; (CLASS(c=getch_internal(),' ') &&
725  ( c != EOF )); i++)
726  {
727  if (c == '\n') linepos++;
728  tok_wspace = check_extend_str(tok_wspace,i,&tok_wspacelen);
729  tok_wspace[i] = c;
730  }
731  tok_wspace[i] = '\0';
732 
733  current_tok.init();
734 
735  if (c != EOF)
736  {
737  current_tok.set_filepos(p_filepos-1);
738 
739  if ((quotes) && // quoted strings (with escapes) are allowed
740  (c == quote))
741  {
742  for (i=0;
743  ((c = getch_internal()) != EOF)
744  ;)
745  {
746  if (c == quote)
747  break;
748  tok_stuff = check_extend_str(tok_stuff,i,&tok_stufflen);
749  if (c == escape)
750  c = getch_internal();
751  tok_stuff[i++] = c;
752  }
753  current_tok.set_quoted(TRUE);
754  }
755  else // standard whitespace separated tokens
756  {
757  for (i=0,tok_stuff[i++]=c;
758  (
759  !CLASS(c,'@') &&
760  !CLASS(c=peekch_internal(),' ') &&
761  !CLASS(c,'@') &&
762  ( c != EOF )) ;)
763  {
764  tok_stuff = check_extend_str(tok_stuff,i,&tok_stufflen);
765  // note, we must have peeked to get here.
766  tok_stuff[i++] = getpeeked_internal();
767  }
768  }
769  tok_stuff[i] = '\0';
770  // Are there any punctuation symbols at the start?
771  for (j=0;
772  ((j < i) && CLASS2(tok_stuff[j], '$', '"'));
773  j++);
774  if ((j > 0) && (j < i)) // there are
775  {
776  tok_prepuncs = check_extend_str(tok_prepuncs,j+1,&tok_prepuncslen);
777  memmove(tok_prepuncs,tok_stuff,j);
778  tok_prepuncs[j] = '\0';
779  current_tok.set_prepunctuation(tok_prepuncs);
780  word=&tok_stuff[j];
781  i-=j; // reduce size by number of prepuncs
782  }
783  else
784  {
785  current_tok.set_prepunctuation(EST_String::Empty);
786  word = tok_stuff;
787  }
788  // Are there any punctuation symbols at the end
789  for (j=i-1;
790  ((j > 0) && CLASS2(word[j],'.','"'));
791  j--);
792  if (word[j+1] != '\0')
793  {
794  current_tok.set_punctuation(&word[j+1]);
795  word[j+1] = '\0';
796  }
797  else
798  current_tok.set_punctuation(EST_String::Empty);
799 
800  current_tok.set_token(word);
801  if (tok_wspace[0] == '\0') // feature paths will have null whitespace
802  current_tok.set_whitespace(EST_String::Empty);
803  else
804  current_tok.set_whitespace(tok_wspace);
805  }
806  else
807  {
808  current_tok.set_token(EST_String::Empty);
809  current_tok.set_whitespace(tok_wspace);
810  current_tok.set_punctuation(EST_String::Empty);
811  current_tok.set_prepunctuation(EST_String::Empty);
812  eof_flag = TRUE;
813  }
814 
815  return current_tok;
816 }
817 
819 {
820  // This doesn't really work if there are blank lines (and you want
821  // to know about them)
822 
823  if ((peek().whitespace().contains("\n")) || eof())
824  return TRUE;
825  else
826  return FALSE;
827 
828 }
829 
831 {
832  if (!peeked_tokp) get();
833  peeked_tokp = TRUE;
834  return current_tok;
835 }
836 
837 EST_String quote_string(const EST_String &s,
838  const EST_String &quote,
839  const EST_String &escape,
840  int force)
841 {
842  // Quotes s always if force true, or iff s contains whitespace,
843  // quotes or escapes force is false
844  // Note quote and escape are assumed to be string of length 1
845  EST_String quoted_form;
846  if ((force) ||
847  (s.contains(quote)) ||
848  (s.contains(escape)) ||
849  (s.contains(RXanywhitespace)) ||
850  (s.length() == 0))
851  {
852  // bigger than the quoted form could ever be
853  int i,j;
854  char *quoted = new char[s.length()*(quote.length()+escape.length())+
855  1+quote.length()+quote.length()];
856  quoted[0] = quote(0);
857  for (i=1,j=0; j < s.length(); j++,i++)
858  {
859  if (s(j) == quote(0))
860  quoted[i++] = escape(0);
861  else if (s(j) == escape(0))
862  quoted[i++] = escape(0);
863  quoted[i] = s(j);
864  }
865  quoted[i++] = quote(0);
866  quoted[i] = '\0';
867  quoted_form = quoted;
868  delete [] quoted;
869  return quoted_form;
870  }
871  else
872  return s;
873 }
874 
876 {
877  return Origin+":"+itoString(linepos);
878 }
~EST_TokenStream()
will close file if appropriate for type
Definition: EST_Token.cc:167
int fread(void *buff, int size, int nitems) EST_WARN_UNUSED_RESULT
Reading binary data, (don&#39;t use peek() immediately beforehand)
Definition: EST_Token.cc:355
void close(void)
Close stream.
Definition: EST_Token.cc:406
int contains(const char *s, int pos=-1) const
Does it contain this substring?
Definition: EST_String.h:375
int open(const EST_String &filename)
open a {EST_TokenStream} for a file.
Definition: EST_Token.cc:200
int quoted() const
TRUE is token was quoted.
Definition: EST_Token.h:169
int open_string(const EST_String &newbuffer)
open a {EST_TokenStream} for string rather than a file
Definition: EST_Token.cc:251
void set_token(const EST_String &p)
set token from a string
Definition: EST_Token.h:95
int restart(void)
Reset to start of file/string.
Definition: EST_Token.cc:437
EST_Token & peek(void)
peek at next token
Definition: EST_Token.cc:830
EST_Token & get()
get next token in stream
Definition: EST_Token.cc:710
EST_Token get_upto(const EST_String &s)
get up to { s} in stream as a single token.
Definition: EST_Token.cc:492
const EST_String pos_description() const
A string describing current position, suitable for error messages.
Definition: EST_Token.cc:107
const EST_String pos_description()
A string describing current position, suitable for error messages.
Definition: EST_Token.cc:875
EST_Token get_upto_eoln(void)
get up to { s} in end of line as a single token.
Definition: EST_Token.cc:516
int seek(int position)
seek, reposition file pointer
Definition: EST_Token.cc:305
static const EST_String Empty
Constant empty string.
Definition: EST_String.h:111
int length(void) const
Length of string ({not} length of underlying chunk)
Definition: EST_String.h:241
int eoln()
end of line
Definition: EST_Token.cc:818