11 /* struct token *tok_get()
19 /* void tok_show_ch(t)
33 /* These functions read from stdin and write to stdout. The
34 /* tokenizer keeps track of where the token appeared in the input
35 /* stream; on output, this information is used to preserve correct
36 /* line number information (even after lots of token lookahead or
37 /* after function-header rewriting) so that diagnostics from the
38 /* next compiler stage make sense.
40 /* tok_get() reads the next token from standard input. It returns
41 /* a null pointer when the end of input is reached.
43 /* tok_show() displays the contents of a (possibly composite) token
44 /* on the standard output.
46 /* tok_show_ch() displays the contents of a single-character token
47 /* on the standard output. The character should not be a newline.
49 /* tok_flush() displays the contents of a (possibly composite) token
50 /* on the standard output and makes it available for re-use.
52 /* put_str() writes a null-terminated string to standard output.
53 /* There should be no newline characters in the string argument.
55 /* put_ch() writes one character to standard output. The character
56 /* should not be a newline.
58 /* put_nl() outputs a newline character and adjusts the program's idea of
59 /* the current output line.
61 /* The in_path and in_line variables contain the file name and
62 /* line number of the most recently read token.
64 /* The tokenizer is just good enough for the unproto filter.
65 /* As a benefit, it is quite fast.
68 /* Eindhoven University of Technology
69 /* Department of Mathematics and Computer Science
70 /* Den Dolech 2, P.O. Box 513, 5600 MB Eindhoven, The Netherlands
77 static char io_sccsid[] = "@(#) tok_io.c 1.3 92/01/15 21:52:59";
84 extern char *strchr();
85 extern char *malloc();
86 extern char *realloc();
87 extern char *strcpy();
89 /* Application-specific stuff */
95 extern char *strsave(); /* XXX need include file */
97 /* Stuff to keep track of original source file name and position */
99 static char def_path[] = ""; /* default path name */
101 char *in_path = def_path; /* current input file name */
102 int in_line = 1; /* current input line number */
104 static char *out_path = def_path; /* last name in output line control */
105 static int out_line = 1; /* current output line number */
106 int last_ch; /* type of last output */
108 /* Forward declarations */
110 static int read_quoted();
111 static void read_comment();
112 static int backslash_newline();
113 static char *read_hex();
114 static char *read_octal();
115 static void fix_line_control();
118 * Character input with one level of pushback. The INPUT() macro recursively
119 * strips backslash-newline pairs from the input stream. The UNPUT() macro
120 * should be used only for characters obtained through the INPUT() macro.
122 * After skipping a backslash-newline pair, the input line counter is not
123 * updated, and we continue with the same logical source line. We just
124 * update a counter with the number of backslash-newline sequences that must
125 * be accounted for (backslash_newline() updates the counter). At the end of
126 * the logical source line, an appropriate number of newline characters is
127 * pushed back (in tok_get()). I do not know how GCC handles this, but it
128 * seems to produce te same output.
130 * Because backslash_newline() recursively calls itself (through the INPUT()
131 * macro), we will run out of stack space, given a sufficiently long
132 * sequence of backslash-newline pairs.
135 static char in_char = 0; /* push-back storage */
136 static int in_flag = 0; /* pushback available */
137 static int nl_compensate = 0; /* line continuation kluge */
139 #define INPUT(c) (in_flag ? (in_flag = 0, c = in_char) : \
140 (c = getchar()) != '\\' ? c : \
141 (c = getchar()) != '\n' ? (ungetc(c, stdin), c = '\\') : \
142 (c = backslash_newline()))
143 #define UNPUT(c) (in_flag = 1, in_char = c)
145 /* Directives that should be ignored. */
147 #ifdef IGNORE_DIRECTIVES
149 static char *ignore_directives[] = {
156 /* Modified string and ctype stuff. */
158 #define STREQUAL(x,y) (*(x) == *(y) && strcmp((x),(y)) == 0)
160 #define ISALNUM(c) (isalnum(c) || (c) == '_')
161 #define ISALPHA(c) (isalpha(c) || (c) == '_')
162 #define ISSPACE(c) (isspace(c) && c != '\n')
163 #define ISDOT(c) (c == '.')
164 #define ISHEX(c) (isdigit(c) || strchr("abcdefABCDEF", c) != 0)
165 #define ISOCTAL(c) (isdigit(c) && (c) != '8' && (c) != '9')
167 /* Collect all characters that satisfy one condition */
169 #define COLLECT(v,c,cond) { \
170 register struct vstring *vs = v; \
171 register char *cp = vs->str; \
173 while (INPUT(c) != EOF) { \
175 if (VS_ADDCH(vs, cp, c) == 0) \
176 fatal("out of memory"); \
185 /* Ensure that output line information is correct */
187 #define CHECK_LINE_CONTROL(p,l) { if (out_path != (p) || out_line != (l)) \
188 fix_line_control((p),(l)); }
190 /* do_control - parse control line */
192 static int do_control()
198 /* Make sure that the directive shows up in the right place. */
200 CHECK_LINE_CONTROL(in_path, in_line);
202 while (t = tok_get()) {
206 /* Ignore blanks after "#" token. */
213 * Line control is of the form: number pathname junk. Since we
214 * have no idea what junk the preprocessor may generate, we copy
215 * all line control tokens to stdout.
219 line = atoi(t->vstr->str); /* extract line number */
221 while ((t = tok_get()) && t->tokno == TOK_WSPACE)
222 tok_flush(t); /* copy white space */
223 if (t) { /* extract path name */
224 path = (t->tokno == '"') ? strsave(t->vstr->str) : in_path;
226 tok_flush(t); /* copy until newline */
227 } while (t->tokno != '\n' && (t = tok_get()));
229 out_line = in_line = line; /* synchronize */
230 out_path = in_path = path; /* synchronize */
233 #ifdef IGNORE_DIRECTIVES
238 * Optionally ignore other #directives. This is only a partial
239 * solution, because the preprocessor will still see them.
243 char *cp = t->vstr->str;
245 for (cpp = ignore_directives; *cpp; cpp++) {
246 if (STREQUAL(cp, *cpp)) {
249 } while (t->tokno != '\n' && (t = tok_get()));
261 } while (t->tokno != '\n' && (t = tok_get()));
272 /* backslash_newline - fix up things after reading a backslash-newline pair */
274 static int backslash_newline()
282 /* tok_get - get next token */
284 static int last_tokno = '\n';
286 struct token *tok_get()
288 register struct token *t;
293 * Get one from the pool and fill it in. The loop is here in case we hit
294 * a preprocessor control line, which happens in a minority of all cases.
295 * We update the token input path and line info *after* backslash-newline
296 * processing or the newline compensation would go wrong.
302 if ((INPUT(c)) == EOF) {
305 } else if ((t->line = in_line, t->path = in_path), !isascii(c)) {
308 t->tokno = TOK_OTHER;
310 } else if (ISSPACE(c)) {
311 COLLECT(t->vstr, c, ISSPACE(c));
312 t->tokno = TOK_WSPACE;
314 } else if (ISALPHA(c)) {
315 COLLECT(t->vstr, c, ISALNUM(c));
318 } else if (isdigit(c)) {
319 COLLECT(t->vstr, c, isdigit(c));
320 t->tokno = TOK_NUMBER;
322 } else if (c == '"' || c == '\'') {
323 t->tokno = read_quoted(t->vstr, c); /* detect missing end quote */
325 } else if (ISDOT(c)) {
326 COLLECT(t->vstr, c, ISDOT(c));
327 t->tokno = TOK_OTHER;
329 } else if (c == '#' && last_tokno == '\n') {
336 if (nl_compensate > 0) { /* compensation for bs-nl */
340 } else if (c == '/') {
341 if ((INPUT(d)) == '*') {
342 t->vstr->str[1] = d; /* comment */
343 read_comment(t->vstr);
344 t->tokno = TOK_WSPACE;
350 } else if (c == '\\') {
351 t->vstr->str[1] = (INPUT(c) == EOF ? 0 : c);
353 t->tokno = TOK_OTHER;
361 last_tokno = t->tokno;
362 t->end_line = in_line;
366 /* read_quoted - read string or character literal, canonicalize escapes */
368 static int read_quoted(vs, ch)
369 register struct vstring *vs;
372 register char *cp = vs->str;
379 * Clobber the token type in case of a premature newline or EOF. This
380 * prevents us from attempting to concatenate string constants with
381 * broken ones that have no closing quote.
384 while (INPUT(c) != EOF) {
385 if (c == '\n') { /* newline in string */
389 if (VS_ADDCH(vs, cp, c) == 0) /* store character */
390 fatal("out of memory");
391 if (c == ch) { /* closing quote */
395 if (c == '\\') { /* parse escape sequence */
396 if ((INPUT(c)) == EOF) { /* EOF, punt */
398 } else if (c == 'a') { /* \a -> audible bell */
399 if ((cp = vs_strcpy(vs, cp, BELL)) == 0)
400 fatal("out of memory");
401 } else if (c == 'x') { /* \xhh -> \nnn */
402 cp = read_hex(vs, cp);
403 } else if (ISOCTAL(c) && ch != '\'') {
404 cp = read_octal(vs, cp, c); /* canonicalize \octal */
406 if (VS_ADDCH(vs, cp, c) == 0) /* \other: leave alone */
407 fatal("out of memory");
415 /* read_comment - stuff a whole comment into one huge token */
417 static void read_comment(vs)
418 register struct vstring *vs;
420 register char *cp = vs->str + 2; /* skip slash star */
424 while (INPUT(c) != EOF) {
425 if (VS_ADDCH(vs, cp, c) == 0)
426 fatal("out of memory");
428 if ((INPUT(d)) == '/') {
429 if (VS_ADDCH(vs, cp, d) == 0)
430 fatal("out of memory");
436 } else if (c == '\n') {
438 } else if (c == '\\') {
439 if ((INPUT(d)) != EOF && VS_ADDCH(vs, cp, d) == 0)
440 fatal("out of memory");
446 /* read_hex - rewrite hex escape to three-digit octal escape */
448 static char *read_hex(vs, cp)
459 * Eat up all subsequent hex digits. Complain later when there are too
463 for (i = 0; i < sizeof(buf) && (INPUT(c) != EOF) && ISHEX(c); i++)
467 if (i < sizeof(buf) && c)
471 * Convert hex form to three-digit octal form. The three-digit form is
472 * used so that strings can be concatenated without problems. Complain
473 * about malformed input; truncate the result to at most three octal
478 error("\\x escape sequence without hexadecimal digits");
479 if (VS_ADDCH(vs, cp, 'x') == 0)
480 fatal("out of memory");
482 (void) sscanf(buf, "%x", &val);
483 sprintf(buf, "%03o", val);
484 if ((len = strlen(buf)) > 3)
485 error("\\x escape sequence yields non-character value");
486 if ((cp = vs_strcpy(vs, cp, buf + len - 3)) == 0)
487 fatal("out of memory");
492 /* read_octal - convert octal escape to three-digit format */
494 static char obuf[] = "00123";
496 static char *read_octal(vs, cp, c)
497 register struct vstring *vs;
503 #define buf_input (obuf + 2)
505 /* Eat up at most three octal digits. */
508 for (i = 1; i < 3 && (INPUT(c) != EOF) && ISOCTAL(c); i++)
516 * Leave three-digit octal escapes alone. Convert one-digit and two-digit
517 * octal escapes to three-digit form by prefixing them with a suitable
518 * number of '0' characters. This is done so that strings can be
519 * concatenated without problems.
522 if ((cp = vs_strcpy(vs, cp, buf_input + i - 3)) == 0)
523 fatal("out of memory");
527 /* put_nl - emit newline and adjust output line count */
535 /* fix_line_control - to adjust path and/or line count info in output */
537 static void fix_line_control(path, line)
543 * This function is called sporadically, so it should not be a problem
544 * that we repeat some of the tests that preceded this function call.
546 * Emit a newline if we are not at the start of a line.
548 * If we switch files, or if we jump backwards, emit line control. If we
549 * jump forward, emit the proper number of newlines to compensate.
552 if (last_ch != '\n') /* terminate open line */
554 if (path != out_path || line < out_line) { /* file switch or back jump */
555 printf("# %d %s\n", out_line = line, out_path = path);
557 } else { /* forward jump */
558 while (line > out_line)
563 /* tok_show_ch - output single-character token (not newline) */
566 register struct token *t;
568 CHECK_LINE_CONTROL(t->path, t->line);
570 put_ch(t->tokno); /* show token contents */
573 /* tok_show - output (possibly composite) token */
576 register struct token *t;
578 register struct token *p;
580 if (t->tokno == TOK_LIST) {
581 register struct token *s;
584 * This branch is completely in terms of tok_xxx() primitives, so
585 * there is no need to check the line control information.
588 for (s = t->head; s; s = s->next) {
589 tok_show_ch(s); /* '(' or ',' or ')' */
590 for (p = s->head; p; p = p->next)
591 tok_show(p); /* show list element */
594 register char *cp = t->vstr->str;
597 * Measurements show that it pays off to give special treatment to
598 * single-character tokens. Note that both types of token may cause a
599 * change of output line number.
602 CHECK_LINE_CONTROL(t->path, t->line);
604 put_ch(*cp); /* single-character token */
606 put_str(cp); /* multi_character token */
608 out_line = t->end_line; /* may span multiple lines */
609 for (p = t->head; p; p = p->next)
610 tok_show(p); /* trailing blanks */