Message ID | fc7836bc2ab8d8eded60098081d74d427091ad74.1703602054.git.liezhi.yang@windriver.com |
---|---|
State | New |
Headers | show |
Series | [v2,1/1] bitbake: event: Improve error message for eventhandler | expand |
On Tue, 2023-12-26 at 07:11 -0800, Robert Yang via lists.openembedded.org wrote: > From: Robert Yang <liezhi.yang@windriver.com> > > * In python 3.10.9, the error message was: > ERROR: Unable to register event handler 'defaultbase_eventhandler': > File "/path/to/poky/meta/classes-global/base.bbclass", line 4 > # SPDX-License-Identifier: MIT > ^^^^^ > SyntaxError: invalid syntax > > This is hard to debug since the error line number 4 is incorrect, but nothing > is wrong with the code in line 4. > > * In python 3.8.10, the error message was: > ERROR: Unable to register event handler 'defaultbase_eventhandler': > File "/path/to/poky/meta/classes-global/base.bbclass", line 4 > an error line > ^ > SyntaxError: invalid syntax > > * Now the error message and lineno is correct: > ERROR: Unable to register event handler 'defaultbase_eventhandler': > File "/path/to/poky/meta/classes-global/base.bbclass", line 256 > an error line > ^^^^^ > SyntaxError: invalid syntax > > * Fixed it via: > - Don't pass filename to compile() to make it can report error code correctly > in both python 3.8.10 and 3.10.9. > > - Fix the lineno offset and filename in traceback message. > > Signed-off-by: Robert Yang <liezhi.yang@windriver.com> > --- > bitbake/lib/bb/event.py | 23 ++++++++++++++++++----- > 1 file changed, 18 insertions(+), 5 deletions(-) > > diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py > index f8acacd80d1..9eecb141379 100644 > --- a/bitbake/lib/bb/event.py > +++ b/bitbake/lib/bb/event.py > @@ -260,16 +260,29 @@ def register(name, handler, mask=None, filename=None, lineno=None, data=None): > try: > code = bb.methodpool.compile_cache(tmp) > if not code: > - if filename is None: > - filename = "%s(e, d)" % name > - code = compile(tmp, filename, "exec", ast.PyCF_ONLY_AST) > + code = compile(tmp, name, "exec", ast.PyCF_ONLY_AST) > if lineno is not None: > ast.increment_lineno(code, lineno-1) > - code = compile(code, filename, "exec") > + code = compile(code, name, "exec") > bb.methodpool.compile_cache_add(tmp, code) > except SyntaxError: > + exc_lines = traceback.format_exc(limit=0).splitlines() > + # Fix lineno offset manually since compile() cannot handle it correctly > + if filename and lineno: > + line_first = exc_lines[0] > + if 'File ' in line_first and ' line ' in line_first: > + lineno_new = line_first.split(' line ')[-1] > + if lineno_new: > + lineno_new = int(lineno_new) > + lineno_new += lineno - 1 > + line_first = 'File "%s", line %d' % (filename, lineno_new) > + # Fix filename > + exc_lines[0] = line_first.replace(name, filename) > + else: > + logger.warning('Failed to fix lineno for %s:%s' % (filename, name)) > + > logger.error("Unable to register event handler '%s':\n%s", name, > - ''.join(traceback.format_exc(limit=0))) > + '\n'.join(exc_lines)) > _handlers[name] = noop > return This is better but I'm still wondering if we can improve. The other trick we could use is to inject "\n" characters into the start of the string you're compiling so the line numbers really match up. Would that work better? Cheers, Richard
Hi RP, On 12/28/23 07:42, Richard Purdie wrote: > On Tue, 2023-12-26 at 07:11 -0800, Robert Yang via > lists.openembedded.org wrote: >> From: Robert Yang <liezhi.yang@windriver.com> >> >> * In python 3.10.9, the error message was: >> ERROR: Unable to register event handler 'defaultbase_eventhandler': >> File "/path/to/poky/meta/classes-global/base.bbclass", line 4 >> # SPDX-License-Identifier: MIT >> ^^^^^ >> SyntaxError: invalid syntax >> >> This is hard to debug since the error line number 4 is incorrect, but nothing >> is wrong with the code in line 4. >> >> * In python 3.8.10, the error message was: >> ERROR: Unable to register event handler 'defaultbase_eventhandler': >> File "/path/to/poky/meta/classes-global/base.bbclass", line 4 >> an error line >> ^ >> SyntaxError: invalid syntax >> >> * Now the error message and lineno is correct: >> ERROR: Unable to register event handler 'defaultbase_eventhandler': >> File "/path/to/poky/meta/classes-global/base.bbclass", line 256 >> an error line >> ^^^^^ >> SyntaxError: invalid syntax >> >> * Fixed it via: >> - Don't pass filename to compile() to make it can report error code correctly >> in both python 3.8.10 and 3.10.9. >> >> - Fix the lineno offset and filename in traceback message. >> >> Signed-off-by: Robert Yang <liezhi.yang@windriver.com> >> --- >> bitbake/lib/bb/event.py | 23 ++++++++++++++++++----- >> 1 file changed, 18 insertions(+), 5 deletions(-) >> >> diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py >> index f8acacd80d1..9eecb141379 100644 >> --- a/bitbake/lib/bb/event.py >> +++ b/bitbake/lib/bb/event.py >> @@ -260,16 +260,29 @@ def register(name, handler, mask=None, filename=None, lineno=None, data=None): >> try: >> code = bb.methodpool.compile_cache(tmp) >> if not code: >> - if filename is None: >> - filename = "%s(e, d)" % name >> - code = compile(tmp, filename, "exec", ast.PyCF_ONLY_AST) >> + code = compile(tmp, name, "exec", ast.PyCF_ONLY_AST) >> if lineno is not None: >> ast.increment_lineno(code, lineno-1) >> - code = compile(code, filename, "exec") >> + code = compile(code, name, "exec") >> bb.methodpool.compile_cache_add(tmp, code) >> except SyntaxError: >> + exc_lines = traceback.format_exc(limit=0).splitlines() >> + # Fix lineno offset manually since compile() cannot handle it correctly >> + if filename and lineno: >> + line_first = exc_lines[0] >> + if 'File ' in line_first and ' line ' in line_first: >> + lineno_new = line_first.split(' line ')[-1] >> + if lineno_new: >> + lineno_new = int(lineno_new) >> + lineno_new += lineno - 1 >> + line_first = 'File "%s", line %d' % (filename, lineno_new) >> + # Fix filename >> + exc_lines[0] = line_first.replace(name, filename) >> + else: >> + logger.warning('Failed to fix lineno for %s:%s' % (filename, name)) >> + >> logger.error("Unable to register event handler '%s':\n%s", name, >> - ''.join(traceback.format_exc(limit=0))) >> + '\n'.join(exc_lines)) >> _handlers[name] = noop >> return > > > This is better but I'm still wondering if we can improve. The other > trick we could use is to inject "\n" characters into the start of the > string you're compiling so the line numbers really match up. Would that > work better? If you meant the following line: if 'File ' in line_first and ' line ' in line_first: I can use re.match() to improve it and send a V3. // Robert > > Cheers, > > Richard >
On Thu, 2023-12-28 at 15:15 +0800, Robert Yang wrote: > Hi RP, > > On 12/28/23 07:42, Richard Purdie wrote: > > On Tue, 2023-12-26 at 07:11 -0800, Robert Yang via > > lists.openembedded.org wrote: > > > From: Robert Yang <liezhi.yang@windriver.com> > > > > > > * In python 3.10.9, the error message was: > > > ERROR: Unable to register event handler 'defaultbase_eventhandler': > > > File "/path/to/poky/meta/classes-global/base.bbclass", line 4 > > > # SPDX-License-Identifier: MIT > > > ^^^^^ > > > SyntaxError: invalid syntax > > > > > > This is hard to debug since the error line number 4 is incorrect, but nothing > > > is wrong with the code in line 4. > > > > > > * In python 3.8.10, the error message was: > > > ERROR: Unable to register event handler 'defaultbase_eventhandler': > > > File "/path/to/poky/meta/classes-global/base.bbclass", line 4 > > > an error line > > > ^ > > > SyntaxError: invalid syntax > > > > > > * Now the error message and lineno is correct: > > > ERROR: Unable to register event handler 'defaultbase_eventhandler': > > > File "/path/to/poky/meta/classes-global/base.bbclass", line 256 > > > an error line > > > ^^^^^ > > > SyntaxError: invalid syntax > > > > > > * Fixed it via: > > > - Don't pass filename to compile() to make it can report error code correctly > > > in both python 3.8.10 and 3.10.9. > > > > > > - Fix the lineno offset and filename in traceback message. > > > > > > Signed-off-by: Robert Yang <liezhi.yang@windriver.com> > > > --- > > > bitbake/lib/bb/event.py | 23 ++++++++++++++++++----- > > > 1 file changed, 18 insertions(+), 5 deletions(-) > > > > > > diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py > > > index f8acacd80d1..9eecb141379 100644 > > > --- a/bitbake/lib/bb/event.py > > > +++ b/bitbake/lib/bb/event.py > > > @@ -260,16 +260,29 @@ def register(name, handler, mask=None, filename=None, lineno=None, data=None): > > > try: > > > code = bb.methodpool.compile_cache(tmp) > > > if not code: > > > - if filename is None: > > > - filename = "%s(e, d)" % name > > > - code = compile(tmp, filename, "exec", ast.PyCF_ONLY_AST) > > > + code = compile(tmp, name, "exec", ast.PyCF_ONLY_AST) > > > if lineno is not None: > > > ast.increment_lineno(code, lineno-1) > > > - code = compile(code, filename, "exec") > > > + code = compile(code, name, "exec") > > > bb.methodpool.compile_cache_add(tmp, code) > > > except SyntaxError: > > > + exc_lines = traceback.format_exc(limit=0).splitlines() > > > + # Fix lineno offset manually since compile() cannot handle it correctly > > > + if filename and lineno: > > > + line_first = exc_lines[0] > > > + if 'File ' in line_first and ' line ' in line_first: > > > + lineno_new = line_first.split(' line ')[-1] > > > + if lineno_new: > > > + lineno_new = int(lineno_new) > > > + lineno_new += lineno - 1 > > > + line_first = 'File "%s", line %d' % (filename, lineno_new) > > > + # Fix filename > > > + exc_lines[0] = line_first.replace(name, filename) > > > + else: > > > + logger.warning('Failed to fix lineno for %s:%s' % (filename, name)) > > > + > > > logger.error("Unable to register event handler '%s':\n%s", name, > > > - ''.join(traceback.format_exc(limit=0))) > > > + '\n'.join(exc_lines)) > > > _handlers[name] = noop > > > return > > > > > > This is better but I'm still wondering if we can improve. The other > > trick we could use is to inject "\n" characters into the start of the > > string you're compiling so the line numbers really match up. Would that > > work better? > > If you meant the following line: > > if 'File ' in line_first and ' line ' in line_first: > > I can use re.match() to improve it and send a V3. I mean something like: tmp = "\n" * (lineno-1) + tmp which would increase the line numbers on the string to match the needed line numbers before we compile it. Cheers, Richard
On 12/28/23 6:44 PM, Richard Purdie wrote: > On Thu, 2023-12-28 at 15:15 +0800, Robert Yang wrote: >> Hi RP, >> >> On 12/28/23 07:42, Richard Purdie wrote: >>> On Tue, 2023-12-26 at 07:11 -0800, Robert Yang via >>> lists.openembedded.org wrote: >>>> From: Robert Yang <liezhi.yang@windriver.com> >>>> >>>> * In python 3.10.9, the error message was: >>>> ERROR: Unable to register event handler 'defaultbase_eventhandler': >>>> File "/path/to/poky/meta/classes-global/base.bbclass", line 4 >>>> # SPDX-License-Identifier: MIT >>>> ^^^^^ >>>> SyntaxError: invalid syntax >>>> >>>> This is hard to debug since the error line number 4 is incorrect, but nothing >>>> is wrong with the code in line 4. >>>> >>>> * In python 3.8.10, the error message was: >>>> ERROR: Unable to register event handler 'defaultbase_eventhandler': >>>> File "/path/to/poky/meta/classes-global/base.bbclass", line 4 >>>> an error line >>>> ^ >>>> SyntaxError: invalid syntax >>>> >>>> * Now the error message and lineno is correct: >>>> ERROR: Unable to register event handler 'defaultbase_eventhandler': >>>> File "/path/to/poky/meta/classes-global/base.bbclass", line 256 >>>> an error line >>>> ^^^^^ >>>> SyntaxError: invalid syntax >>>> >>>> * Fixed it via: >>>> - Don't pass filename to compile() to make it can report error code correctly >>>> in both python 3.8.10 and 3.10.9. >>>> >>>> - Fix the lineno offset and filename in traceback message. >>>> >>>> Signed-off-by: Robert Yang <liezhi.yang@windriver.com> >>>> --- >>>> bitbake/lib/bb/event.py | 23 ++++++++++++++++++----- >>>> 1 file changed, 18 insertions(+), 5 deletions(-) >>>> >>>> diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py >>>> index f8acacd80d1..9eecb141379 100644 >>>> --- a/bitbake/lib/bb/event.py >>>> +++ b/bitbake/lib/bb/event.py >>>> @@ -260,16 +260,29 @@ def register(name, handler, mask=None, filename=None, lineno=None, data=None): >>>> try: >>>> code = bb.methodpool.compile_cache(tmp) >>>> if not code: >>>> - if filename is None: >>>> - filename = "%s(e, d)" % name >>>> - code = compile(tmp, filename, "exec", ast.PyCF_ONLY_AST) >>>> + code = compile(tmp, name, "exec", ast.PyCF_ONLY_AST) >>>> if lineno is not None: >>>> ast.increment_lineno(code, lineno-1) >>>> - code = compile(code, filename, "exec") >>>> + code = compile(code, name, "exec") >>>> bb.methodpool.compile_cache_add(tmp, code) >>>> except SyntaxError: >>>> + exc_lines = traceback.format_exc(limit=0).splitlines() >>>> + # Fix lineno offset manually since compile() cannot handle it correctly >>>> + if filename and lineno: >>>> + line_first = exc_lines[0] >>>> + if 'File ' in line_first and ' line ' in line_first: >>>> + lineno_new = line_first.split(' line ')[-1] >>>> + if lineno_new: >>>> + lineno_new = int(lineno_new) >>>> + lineno_new += lineno - 1 >>>> + line_first = 'File "%s", line %d' % (filename, lineno_new) >>>> + # Fix filename >>>> + exc_lines[0] = line_first.replace(name, filename) >>>> + else: >>>> + logger.warning('Failed to fix lineno for %s:%s' % (filename, name)) >>>> + >>>> logger.error("Unable to register event handler '%s':\n%s", name, >>>> - ''.join(traceback.format_exc(limit=0))) >>>> + '\n'.join(exc_lines)) >>>> _handlers[name] = noop >>>> return >>> >>> >>> This is better but I'm still wondering if we can improve. The other >>> trick we could use is to inject "\n" characters into the start of the >>> string you're compiling so the line numbers really match up. Would that >>> work better? >> >> If you meant the following line: >> >> if 'File ' in line_first and ' line ' in line_first: >> >> I can use re.match() to improve it and send a V3. > > I mean something like: > > tmp = "\n" * (lineno-1) + tmp > > which would increase the line numbers on the string to match the needed > line numbers before we compile it. Cool, this is very simple and works well, V3 is coming. // Robert > > Cheers, > > Richard
diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py index f8acacd80d1..9eecb141379 100644 --- a/bitbake/lib/bb/event.py +++ b/bitbake/lib/bb/event.py @@ -260,16 +260,29 @@ def register(name, handler, mask=None, filename=None, lineno=None, data=None): try: code = bb.methodpool.compile_cache(tmp) if not code: - if filename is None: - filename = "%s(e, d)" % name - code = compile(tmp, filename, "exec", ast.PyCF_ONLY_AST) + code = compile(tmp, name, "exec", ast.PyCF_ONLY_AST) if lineno is not None: ast.increment_lineno(code, lineno-1) - code = compile(code, filename, "exec") + code = compile(code, name, "exec") bb.methodpool.compile_cache_add(tmp, code) except SyntaxError: + exc_lines = traceback.format_exc(limit=0).splitlines() + # Fix lineno offset manually since compile() cannot handle it correctly + if filename and lineno: + line_first = exc_lines[0] + if 'File ' in line_first and ' line ' in line_first: + lineno_new = line_first.split(' line ')[-1] + if lineno_new: + lineno_new = int(lineno_new) + lineno_new += lineno - 1 + line_first = 'File "%s", line %d' % (filename, lineno_new) + # Fix filename + exc_lines[0] = line_first.replace(name, filename) + else: + logger.warning('Failed to fix lineno for %s:%s' % (filename, name)) + logger.error("Unable to register event handler '%s':\n%s", name, - ''.join(traceback.format_exc(limit=0))) + '\n'.join(exc_lines)) _handlers[name] = noop return env = {}