From 154dd24a549c7f8df1280eae997a4671e01b46fb Mon Sep 17 00:00:00 2001 From: Thibault Kruse Date: Wed, 7 Sep 2016 21:19:33 +0900 Subject: [PATCH 1/4] improve cpplint check by invoking python just once. Reduces time from 1m to 6s --- scripts/python/Makefile.in | 6 ++---- scripts/python/cpplint_wrap.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 scripts/python/cpplint_wrap.py diff --git a/scripts/python/Makefile.in b/scripts/python/Makefile.in index 46e809f..4ecff78 100644 --- a/scripts/python/Makefile.in +++ b/scripts/python/Makefile.in @@ -11,8 +11,6 @@ CXX_SRCS := $(wildcard *.cpp) CXX_LINT := ${CXX_SRCS:.cpp=.lint} .PHONY: cpplint-all -cpplint-all: $(CXX_LINT) - -%.lint: %.cpp - @python ../../python/cpplint.py --verbose=0 --linelength=100 --filter=-legal/copyright,-build/include_order,-build/c++11,-build/namespaces,-build/class,-build/include,-build/include_subdir,-readability/inheritance,-readability/function,-readability/casting,-readability/namespace,-readability/alt_tokens,-readability/braces,-readability/fn_size,-whitespace/comments,-whitespace/braces,-whitespace/empty_loop_body,-whitespace/indent,-whitespace/newline,-runtime/explicit,-runtime/arrays,-runtime/int,-runtime/references,-runtime/string,-runtime/operator $< || (cat $< | nl -ba | grep -v 'md-split' && false) +cpplint-all: + @python ../../python/cpplint_wrap.py *.cpp diff --git a/scripts/python/cpplint_wrap.py b/scripts/python/cpplint_wrap.py new file mode 100644 index 0000000..fd7a9f9 --- /dev/null +++ b/scripts/python/cpplint_wrap.py @@ -0,0 +1,32 @@ +## wraps local cpplint to produce verbose output without code harness +import cpplint +import sys + +def main(): + FILTERS='cpplint --verbose=0 --linelength=100 --filter=-legal/copyright,-build/include_order,-build/c++11,-build/namespaces,-build/class,-build/include,-build/include_subdir,-readability/inheritance,-readability/function,-readability/casting,-readability/namespace,-readability/alt_tokens,-readability/braces,-readability/fn_size,-whitespace/comments,-whitespace/braces,-whitespace/empty_loop_body,-whitespace/indent,-whitespace/newline,-runtime/explicit,-runtime/arrays,-runtime/int,-runtime/references,-runtime/string,-runtime/operator'.split(' ') + + result = False + files = sys.argv[1:] + for loopfile in files: + newargs = FILTERS + [loopfile] + sys.argv = newargs + + try: + cpplint.main() + except SystemExit as e: + last_result = e.args[0] + result = result or last_result + if (last_result): + write_code_lines(loopfile) + sys.exit(result) + +def write_code_lines(filename): + with open(filename, 'r') as f: + linenum = 1 + for line in f: + if (not '// by md-split' in line): + sys.stdout.write('%3d %s' % (linenum, line)) + linenum += 1 + +if __name__ == '__main__': + main() From 8e282fa90c7b159bc93e725a1d529c935b67c73b Mon Sep 17 00:00:00 2001 From: Thibault Kruse Date: Sat, 10 Sep 2016 09:05:44 +0900 Subject: [PATCH 2/4] Minor fix to python code extractor, dedent by indentation amount --- scripts/python/md-split.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/python/md-split.py b/scripts/python/md-split.py index fb147e9..21af9d9 100755 --- a/scripts/python/md-split.py +++ b/scripts/python/md-split.py @@ -105,7 +105,7 @@ def process_code(read_filehandle, text_filehandle, line, linenum, sourcefile, co if (not line.strip() == '```'): if ('???' == no_comment_line or '...' == no_comment_line): has_question_marks = True - linebuffer.append(dedent(line) if not fenced else line) + linebuffer.append(dedent(line, indent_depth) if not fenced else line) try: line = read_filehandle.next() linenum += 1 @@ -147,7 +147,7 @@ def is_code(line, indent_depth = 4): return 0 def is_inside_code(line, indent_depth): - return is_code(line, indent_depth) or line.strip() == '' + return is_code(line, indent_depth) > 0 or line.strip() == '' def stripped(line): # Remove well-formed html tags, fixing mistakes by legitimate users @@ -155,9 +155,9 @@ def stripped(line): sline = re.sub('[()\[\]#*]', ' ', line) return sline -def dedent(line): - if line.startswith(' '): - return line[4:] +def dedent(line, indent_depth): + if line.startswith(' ' * indent_depth): + return line[indent_depth:] if line.startswith('\t'): return line[1:] return line From 1842b88c21eaab515eadaa0cb359763d8ee296e9 Mon Sep 17 00:00:00 2001 From: Thibault Kruse Date: Sat, 10 Sep 2016 09:09:37 +0900 Subject: [PATCH 3/4] refactor python code extractor: extract function --- scripts/python/md-split.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/scripts/python/md-split.py b/scripts/python/md-split.py index 21af9d9..e06fe85 100755 --- a/scripts/python/md-split.py +++ b/scripts/python/md-split.py @@ -115,10 +115,19 @@ def process_code(read_filehandle, text_filehandle, line, linenum, sourcefile, co codefile = os.path.join(codedir, '%s%s.cpp' % (name, index)) if fenced: text_filehandle.write('\n') + if (has_actual_code and not has_question_marks): - # add commonly used headers, so that lines can compile - with io.open(codefile, 'w') as code_filehandle: - code_filehandle.write('''\ + write_with_harness(codefile, sourcefile, start_linenum, linebuffer) + return (line, linenum) + + +def write_with_harness(codefile, sourcefile, start_linenum, linebuffer): + '''write output with additional lines to make code likely compilable''' + # add commonly used headers, so that lines can likely compile. + # This is work in progress, the main issue remains handling class + # declarations in in-function code differently + with io.open(codefile, 'w') as code_filehandle: + code_filehandle.write('''\ #include // by md-split #include // by md-split #include // by md-split @@ -134,10 +143,9 @@ def process_code(read_filehandle, text_filehandle, line, linenum, sourcefile, co using namespace std; // by md-split // %s : %s ''' % (sourcefile, start_linenum)) - # TODO: if not toplevel code, wrap inside class - for codeline in linebuffer: - code_filehandle.write(codeline) - return (line, linenum) + # TODO: if not toplevel code, wrap inside class + for codeline in linebuffer: + code_filehandle.write(codeline) def is_code(line, indent_depth = 4): From 084eb29d426688617b6e19846f508b6d1cb3a564 Mon Sep 17 00:00:00 2001 From: Thibault Kruse Date: Sat, 10 Sep 2016 09:16:57 +0900 Subject: [PATCH 4/4] minor improvement, strip newlines from end of code snippets --- scripts/python/md-split.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/python/md-split.py b/scripts/python/md-split.py index e06fe85..9a855fc 100755 --- a/scripts/python/md-split.py +++ b/scripts/python/md-split.py @@ -117,10 +117,24 @@ def process_code(read_filehandle, text_filehandle, line, linenum, sourcefile, co text_filehandle.write('\n') if (has_actual_code and not has_question_marks): + linebuffer = clean_trailing_newlines(linebuffer) write_with_harness(codefile, sourcefile, start_linenum, linebuffer) return (line, linenum) +def clean_trailing_newlines(linebuffer): + result = [] + code_started = False + linebuffer.reverse() + for line in linebuffer: + if not code_started and line == '\n': + continue + code_started = True + result.append(line) + result.reverse() + return result + + def write_with_harness(codefile, sourcefile, start_linenum, linebuffer): '''write output with additional lines to make code likely compilable''' # add commonly used headers, so that lines can likely compile.