Skip to content

tokenizer: Remove unused tabs options#4422

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vstinner merged 1 commit into from
Nov 17, 2017
Merged

tokenizer: Remove unused tabs options #4422

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 11 additions & 31 deletions Parser/tokenizer.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -18,6 +18,9 @@
#include "abstract.h"
#endif /* PGEN */

/* Alternate tab spacing */
#define ALTTABSIZE 1

#define is_potential_identifier_start(c) (\
(c >= 'a' && c <= 'z')\
|| (c >= 'A' && c <= 'Z')\
Expand DownExpand Up@@ -133,9 +136,6 @@ tok_new(void)
tok->prompt = tok->nextprompt = NULL;
tok->lineno = 0;
tok->level = 0;
tok->altwarning = 1;
tok->alterror = 1;
tok->alttabsize = 1;
tok->altindstack[0] = 0;
tok->decoding_state = STATE_INIT;
tok->decoding_erred = 0;
Expand DownExpand Up@@ -1283,22 +1283,9 @@ PyToken_ThreeChars(int c1, int c2, int c3)
static int
indenterror(struct tok_state *tok)
{
if (tok->alterror) {
tok->done = E_TABSPACE;
tok->cur = tok->inp;
return 1;
}
if (tok->altwarning) {
#ifdef PGEN
PySys_WriteStderr("inconsistent use of tabs and spaces "
"in indentation\n");
#else
PySys_FormatStderr("%U: inconsistent use of tabs and spaces "
"in indentation\n", tok->filename);
#endif
tok->altwarning = 0;
}
return 0;
tok->done = E_TABSPACE;
tok->cur = tok->inp;
return ERRORTOKEN;
}

#ifdef PGEN
Expand DownExpand Up@@ -1378,9 +1365,8 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
col++, altcol++;
}
else if (c == '\t') {
col = (col/tok->tabsize + 1) * tok->tabsize;
altcol = (altcol/tok->alttabsize + 1)
* tok->alttabsize;
col = (col / tok->tabsize + 1) * tok->tabsize;
altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;

@serhiy-storchakaserhiy-storchakaNov 16, 2017

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is just altcol++, isn't?

@vstinnervstinnerNov 16, 2017

Copy link
Copy Markdown
MemberAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't know well this code, so I prefer to minimize my changes to avoid any risk of regression.

It looks like the code doesn't round properly for tabsize=1. But I was too lazy to try to understand this old code. I prefer to leave it as it is if something has to change the code later.

FYI I found this code when I wanted to enable tabs/spaces warning in my new developer mode... Then I understood that the warning is always emitted and it's even an hard error...

@serhiy-storchakaserhiy-storchakaNov 17, 2017

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I recently learned a new term: Chesterton's fence 😄.

Yes, since we are not going to allow mixing spaces and tabs, altwarning and alterror can be removed. But the algorithm not always works properly (see bpo-32053), and I think that it would be better at least to write it in simpler way.

@vstinnervstinnerNov 17, 2017

Copy link
Copy Markdown
MemberAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please go ahead and continue to cleanup this code ;-) Be for my side, I'm done. As I wrote, I only cared when the syntax warning was emitted (and it's now always emitted).

@serhiy-storchakaserhiy-storchakaNov 17, 2017

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would prefer to not introduce useless and misleading parameters like ALTTABSIZE.

And, to be pedantic, the syntax warning is now never emitted (the syntax error is emitted instead).

}
else if (c == '\014') {/* Control-L (formfeed) */
col = altcol = 0; /* For Emacs users */
Expand DownExpand Up@@ -1409,9 +1395,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
if (col == tok->indstack[tok->indent]) {
/* No change */
if (altcol != tok->altindstack[tok->indent]) {
if (indenterror(tok)) {
return ERRORTOKEN;
}
return indenterror(tok);
}
}
else if (col > tok->indstack[tok->indent]) {
Expand All@@ -1422,9 +1406,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
return ERRORTOKEN;
}
if (altcol <= tok->altindstack[tok->indent]) {
if (indenterror(tok)) {
return ERRORTOKEN;
}
return indenterror(tok);
}
tok->pendin++;
tok->indstack[++tok->indent] = col;
Expand All@@ -1443,9 +1425,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
return ERRORTOKEN;
}
if (altcol != tok->altindstack[tok->indent]) {
if (indenterror(tok)) {
return ERRORTOKEN;
}
return indenterror(tok);
}
}
}
Expand Down
3 changes: 0 additions & 3 deletions Parser/tokenizer.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -47,9 +47,6 @@ struct tok_state {
(Grammar/Grammar). */
PyObject *filename;
#endif
int altwarning; /* Issue warning if alternate tabs don't match */
int alterror; /* Issue error if alternate tabs don't match */
int alttabsize; /* Alternate tab spacing */
int altindstack[MAXINDENT]; /* Stack of alternate indents */
/* Stuff for PEP 0263 */
enum decoding_state decoding_state;
Expand Down