From 52926c03f1a3c0abd5d3f75db3517f10729c6f8a Mon Sep 17 00:00:00 2001 From: cyndyishida Date: Tue, 30 Jan 2018 19:53:48 -0500 Subject: [PATCH 1/7] Initial Integration Test Passing - a single file compilation ran with Pepper then executed and compared to a normal compilation stage - 1 test with no preprocessing statements - also added mac os environment paths --- src/pepper/symbol_table.py | 13 +++++- tests/integration_test.py | 43 +++++++++++++++++++ .../test_data/no_preprocessing_statements.cpp | 6 +++ 3 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 tests/integration_test.py create mode 100644 tests/test_data/no_preprocessing_statements.cpp diff --git a/src/pepper/symbol_table.py b/src/pepper/symbol_table.py index e4281c9..9d74f16 100644 --- a/src/pepper/symbol_table.py +++ b/src/pepper/symbol_table.py @@ -17,13 +17,24 @@ "/usr/lib/gcc/x86_64-linux-gnu/7/include", "/usr/local/include", "/usr/lib/gcc/x86_64-linux-gnu/7/include-fixed", - "/usr/include/x86_64-linux-gnu", + "/usr/include/x86_s4-linux-gnu", "/usr/include" ] +MAC_DEFAULTS = { + "/usr/local/include", + "/Library/Developer/CommandLineTools/usr/include/c++/v1", + "/Library/Developer/CommandLineTools/usr/lib/clang/9.0.0/include", + "/Library/Developer/CommandLineTools/usr/include", + "/usr/include" + +} + if platform.system() == "Linux": SYSTEM_INCLUDE_PATHS = LINUX_DEFAULTS +elif platform.system() == "Darwin": + SYSTEM_INCLUDE_PATHS = MAC_DEFAULTS class MacroExpansion(): def __init__(self, name, expansion, args=None): diff --git a/tests/integration_test.py b/tests/integration_test.py new file mode 100644 index 0000000..7ec4ef0 --- /dev/null +++ b/tests/integration_test.py @@ -0,0 +1,43 @@ +import subprocess as sp +import shutil + + +SOURCE_FILE_DIRECTORY = "./tests/test_data/" +EXAMPLE_OUTPUT_DIRECTORY = "./tests/test_data/output_examples/" +CXX_FLAG = "clang" + + +class TestSystem: + def test_no_preprocessing_statements(self,tmpdir): + curr_file = "/no_preprocessing_statements.cpp" + test_dir = tmpdir.mkdir('compiled') + initial_file = SOURCE_FILE_DIRECTORY + curr_file + + #run Pepper output extention to (.ii) + test_dir_path = test_dir.realpath()+(curr_file[:-3]+ 'ii' ) + process = sp.run(["Pepper", initial_file, "--output_file", test_dir_path], stdout=sp.PIPE) + assert(process.returncode == 0) + + pepper_executable = test_dir.realpath() + "/no_preprocessing_statements.pepper" + + #run compiler using C++ 17 standard + process = sp.run([CXX_FLAG, "-std=c++1z","-o" , pepper_executable ,test_dir_path], + stdout=sp.PIPE, stderr= sp.PIPE) + assert(process.returncode == 0) + + # run executable + pepper_process = sp.Popen([pepper_executable], stdout=sp.PIPE, stderr= sp.PIPE) + p_out,p_err = pepper_process.communicate() + + + + # run normal compilation stage + compile_executable = test_dir.realpath() + "no_preprocessing_statements." + CXX_FLAG + process = sp.run(['g++', "-std=c++1z","-o" , compile_executable , initial_file], + stdout=sp.PIPE, stderr= sp.PIPE) + assert(process.returncode == 0) + + compile_process = sp.Popen([compile_executable], stdout=sp.PIPE, stderr= sp.PIPE) + c_out,c_err = compile_process.communicate() + + assert(p_out == c_out) diff --git a/tests/test_data/no_preprocessing_statements.cpp b/tests/test_data/no_preprocessing_statements.cpp new file mode 100644 index 0000000..39da174 --- /dev/null +++ b/tests/test_data/no_preprocessing_statements.cpp @@ -0,0 +1,6 @@ +int x = 3; + +int main(int argc, char **argv) +{ + return 0; +} \ No newline at end of file From 707d466528430dd71b503e0b8fbd0705e16c85fa Mon Sep 17 00:00:00 2001 From: cyndyishida Date: Sat, 20 Jan 2018 16:14:57 -0500 Subject: [PATCH 2/7] lex token: if directive --- src/pepper/lexer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/pepper/lexer.py b/src/pepper/lexer.py index a10703d..8e48a3b 100755 --- a/src/pepper/lexer.py +++ b/src/pepper/lexer.py @@ -30,6 +30,7 @@ 'ifndef', 'endif', 'else', + 'if', 'py', ] @@ -78,6 +79,8 @@ def t_PREPROCESSING_KEYWORD_ENDIF(t): r'\#endif\b' return t +def t_PREPROCESSING_KEYWORD_IF(t): + r'\#if\b' def t_PREPROCESSING_KEYWORD_ELSE(t): r'\#else\b' From 1884487ac956d3982cec633db9a0acd1b811651e Mon Sep 17 00:00:00 2001 From: cyndyishida Date: Wed, 31 Jan 2018 12:36:30 -0500 Subject: [PATCH 3/7] Modularized CXX_FLAG - Compiling now with C++11 should be compatiable with g++ 4.8.2 on travis - fixed typo in linux default system paths --- src/pepper/symbol_table.py | 2 +- tests/integration_test.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pepper/symbol_table.py b/src/pepper/symbol_table.py index 9d74f16..88b3f44 100644 --- a/src/pepper/symbol_table.py +++ b/src/pepper/symbol_table.py @@ -17,7 +17,7 @@ "/usr/lib/gcc/x86_64-linux-gnu/7/include", "/usr/local/include", "/usr/lib/gcc/x86_64-linux-gnu/7/include-fixed", - "/usr/include/x86_s4-linux-gnu", + "/usr/include/x86_64-linux-gnu", "/usr/include" ] diff --git a/tests/integration_test.py b/tests/integration_test.py index 7ec4ef0..58f89ca 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -4,7 +4,7 @@ SOURCE_FILE_DIRECTORY = "./tests/test_data/" EXAMPLE_OUTPUT_DIRECTORY = "./tests/test_data/output_examples/" -CXX_FLAG = "clang" +CXX_FLAG = "g++" class TestSystem: @@ -20,8 +20,8 @@ def test_no_preprocessing_statements(self,tmpdir): pepper_executable = test_dir.realpath() + "/no_preprocessing_statements.pepper" - #run compiler using C++ 17 standard - process = sp.run([CXX_FLAG, "-std=c++1z","-o" , pepper_executable ,test_dir_path], + #run compiler using C++ 11 standard + process = sp.run([CXX_FLAG, "-std=c++11","-o" , pepper_executable ,test_dir_path], stdout=sp.PIPE, stderr= sp.PIPE) assert(process.returncode == 0) @@ -33,7 +33,7 @@ def test_no_preprocessing_statements(self,tmpdir): # run normal compilation stage compile_executable = test_dir.realpath() + "no_preprocessing_statements." + CXX_FLAG - process = sp.run(['g++', "-std=c++1z","-o" , compile_executable , initial_file], + process = sp.run([CXX_FLAG, "-std=c++11","-o" , compile_executable , initial_file], stdout=sp.PIPE, stderr= sp.PIPE) assert(process.returncode == 0) From c841fe13b6d462d207ecc9bb4424df5167af9262 Mon Sep 17 00:00:00 2001 From: cyndyishida Date: Tue, 6 Feb 2018 13:22:51 -0500 Subject: [PATCH 4/7] Passing unit tests w/ compiler tests - added blog post - simple bash script to run a single test - local passing unit tests --- .../2018-01-30-building-integration-tests.md | 43 +++++++++++++++++++ run-test | 3 ++ src/pepper/symbol_table.py | 4 +- tests/integration_test.py | 18 ++++---- 4 files changed, 58 insertions(+), 10 deletions(-) create mode 100644 docs/_posts/2018-01-30-building-integration-tests.md create mode 100755 run-test diff --git a/docs/_posts/2018-01-30-building-integration-tests.md b/docs/_posts/2018-01-30-building-integration-tests.md new file mode 100644 index 0000000..7d6877d --- /dev/null +++ b/docs/_posts/2018-01-30-building-integration-tests.md @@ -0,0 +1,43 @@ +--- +layout: post +title: "Building Integration Tests" +date: 2018-01-30 8:50:00 -0400 +categories: devblog research cmi +--- + +_written by [Cyndy Ishida](https://github.com/cyndyishida)_ + +Building Integration Tests is my first notable contribution to the Pepper Project. +I'm hoping that the tests I design will later be the way we actually pipeline Pepper into a normal compiler. +Currently the chain of steps is to run a command like: + +~~~ +Pepper some_file.cpp --output_file some_file.ii +~~~ + +Here it invokes Pepper and runs the output of the preprocessor to the same file name with +the extension of 'ii' which is just an extension that signals to the compiler to skip the pre-processing stage. + +Only in the GNU compiler, you can actually avoid changing the file name with using the '-fpreprocessed' flag. + +Based on a global variable 'CXX_FLAG' which I'm hoping in the future to read the environment variable for it +when it's being ran for either G++ or Clang. I don't see there being a huge need to support MSVC compilers, +but wht do I know. +~~~ +CXX_FLAG -Wall -std=c++17 -o output some_file.ii +~~~ + +Currently I run that to build the pepper-ed executable and also run the original c++ file to the normal compilation + stage and compare both standard outputs, as per Jake's suggestion. + + + ###Concerns for Pepper with Compilation +I would like to actually run Pepper in production and redirect the output from the compiler to the Pepper interface. +This would be pretty simple with the subprocess module and storing the information from standard out and error. +A few things that trouble me about my current stream of execution moving to production is portability. I don't really see + spawning a ton of threads as a viable way of pipeline-ing Pepper, especially if this would ever be used in a memory and + processor(s) intensive situation. I think it would be worth looking into. + + ###GPU Support + As stated above, I really only thought about using Pepper with clang or g++ but applying NVICC support would be cool. + Well maybe just, modularized enough to support any C++ compiler. \ No newline at end of file diff --git a/run-test b/run-test new file mode 100755 index 0000000..3bdac9e --- /dev/null +++ b/run-test @@ -0,0 +1,3 @@ +#!/bin/sh +echo "Running py.test on $1" +py.test -vv --cov-report=html --cov-report=xml --cov=pepper $1 diff --git a/src/pepper/symbol_table.py b/src/pepper/symbol_table.py index 88b3f44..07625f0 100644 --- a/src/pepper/symbol_table.py +++ b/src/pepper/symbol_table.py @@ -21,14 +21,14 @@ "/usr/include" ] -MAC_DEFAULTS = { +MAC_DEFAULTS = [ "/usr/local/include", "/Library/Developer/CommandLineTools/usr/include/c++/v1", "/Library/Developer/CommandLineTools/usr/lib/clang/9.0.0/include", "/Library/Developer/CommandLineTools/usr/include", "/usr/include" -} +] if platform.system() == "Linux": SYSTEM_INCLUDE_PATHS = LINUX_DEFAULTS diff --git a/tests/integration_test.py b/tests/integration_test.py index 58f89ca..58cf9ae 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -7,21 +7,23 @@ CXX_FLAG = "g++" + + class TestSystem: def test_no_preprocessing_statements(self,tmpdir): - curr_file = "/no_preprocessing_statements.cpp" + curr_file = "/no_preprocessing_statements." test_dir = tmpdir.mkdir('compiled') - initial_file = SOURCE_FILE_DIRECTORY + curr_file + initial_file = SOURCE_FILE_DIRECTORY + curr_file + "cpp" #run Pepper output extention to (.ii) - test_dir_path = test_dir.realpath()+(curr_file[:-3]+ 'ii' ) - process = sp.run(["Pepper", initial_file, "--output_file", test_dir_path], stdout=sp.PIPE) + test_file_path = test_dir.realpath()+ curr_file + 'ii' + process = sp.run(["Pepper", initial_file, "--output_file", test_file_path], stdout=sp.PIPE) assert(process.returncode == 0) - pepper_executable = test_dir.realpath() + "/no_preprocessing_statements.pepper" + pepper_executable = test_dir.realpath() + curr_file + "pepper" #run compiler using C++ 11 standard - process = sp.run([CXX_FLAG, "-std=c++11","-o" , pepper_executable ,test_dir_path], + process = sp.run([CXX_FLAG, "-std=c++11","-Wall","-o" , pepper_executable ,test_file_path], stdout=sp.PIPE, stderr= sp.PIPE) assert(process.returncode == 0) @@ -32,8 +34,8 @@ def test_no_preprocessing_statements(self,tmpdir): # run normal compilation stage - compile_executable = test_dir.realpath() + "no_preprocessing_statements." + CXX_FLAG - process = sp.run([CXX_FLAG, "-std=c++11","-o" , compile_executable , initial_file], + compile_executable = test_dir.realpath() + curr_file + CXX_FLAG + process = sp.run([CXX_FLAG, "-std=c++11","-Wall" ,"-o" , compile_executable , initial_file], stdout=sp.PIPE, stderr= sp.PIPE) assert(process.returncode == 0) From 615d1a99b4b1d37f6912eba95bdfe3d072b2b0f3 Mon Sep 17 00:00:00 2001 From: cyndyishida Date: Tue, 6 Feb 2018 13:49:56 -0500 Subject: [PATCH 5/7] Coding standard coverage passing --- src/pepper/lexer.py | 1 + tests/integration_test.py | 25 ++++++++++--------------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/pepper/lexer.py b/src/pepper/lexer.py index 8e48a3b..0661aaa 100755 --- a/src/pepper/lexer.py +++ b/src/pepper/lexer.py @@ -79,6 +79,7 @@ def t_PREPROCESSING_KEYWORD_ENDIF(t): r'\#endif\b' return t + def t_PREPROCESSING_KEYWORD_IF(t): r'\#if\b' diff --git a/tests/integration_test.py b/tests/integration_test.py index 58cf9ae..969ac6a 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -1,5 +1,4 @@ import subprocess as sp -import shutil SOURCE_FILE_DIRECTORY = "./tests/test_data/" @@ -7,39 +6,35 @@ CXX_FLAG = "g++" - - class TestSystem: def test_no_preprocessing_statements(self,tmpdir): curr_file = "/no_preprocessing_statements." test_dir = tmpdir.mkdir('compiled') initial_file = SOURCE_FILE_DIRECTORY + curr_file + "cpp" - #run Pepper output extention to (.ii) - test_file_path = test_dir.realpath()+ curr_file + 'ii' + # run Pepper output extention to (.ii) + test_file_path = test_dir.realpath() + curr_file + 'ii' process = sp.run(["Pepper", initial_file, "--output_file", test_file_path], stdout=sp.PIPE) assert(process.returncode == 0) pepper_executable = test_dir.realpath() + curr_file + "pepper" - #run compiler using C++ 11 standard - process = sp.run([CXX_FLAG, "-std=c++11","-Wall","-o" , pepper_executable ,test_file_path], + # run compiler using C++ 11 standard + process = sp.run([CXX_FLAG, "-std=c++11", "-Wall", "-o", pepper_executable, test_file_path], stdout=sp.PIPE, stderr= sp.PIPE) assert(process.returncode == 0) # run executable - pepper_process = sp.Popen([pepper_executable], stdout=sp.PIPE, stderr= sp.PIPE) - p_out,p_err = pepper_process.communicate() - - + pepper_process = sp.Popen([pepper_executable], stdout=sp.PIPE, stderr=sp.PIPE) + p_out, p_err = pepper_process.communicate() # run normal compilation stage - compile_executable = test_dir.realpath() + curr_file + CXX_FLAG - process = sp.run([CXX_FLAG, "-std=c++11","-Wall" ,"-o" , compile_executable , initial_file], - stdout=sp.PIPE, stderr= sp.PIPE) + compile_executable = test_dir.realpath() + curr_file + CXX_FLAG + process = sp.run([CXX_FLAG, "-std=c++11", "-Wall", "-o", compile_executable, initial_file], + stdout=sp.PIPE, stderr=sp.PIPE) assert(process.returncode == 0) - compile_process = sp.Popen([compile_executable], stdout=sp.PIPE, stderr= sp.PIPE) + compile_process = sp.Popen([compile_executable], stdout=sp.PIPE, stderr=sp.PIPE) c_out,c_err = compile_process.communicate() assert(p_out == c_out) From 521743bb519db079c63332550d2ca70090b832aa Mon Sep 17 00:00:00 2001 From: cyndyishida Date: Tue, 6 Feb 2018 13:57:33 -0500 Subject: [PATCH 6/7] Coverage Now Passing --- tests/integration_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration_test.py b/tests/integration_test.py index 969ac6a..fcd70fd 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -7,7 +7,7 @@ class TestSystem: - def test_no_preprocessing_statements(self,tmpdir): + def test_no_preprocessing_statements(self, tmpdir): curr_file = "/no_preprocessing_statements." test_dir = tmpdir.mkdir('compiled') initial_file = SOURCE_FILE_DIRECTORY + curr_file + "cpp" @@ -21,7 +21,7 @@ def test_no_preprocessing_statements(self,tmpdir): # run compiler using C++ 11 standard process = sp.run([CXX_FLAG, "-std=c++11", "-Wall", "-o", pepper_executable, test_file_path], - stdout=sp.PIPE, stderr= sp.PIPE) + stdout=sp.PIPE, stderr=sp.PIPE) assert(process.returncode == 0) # run executable @@ -35,6 +35,6 @@ def test_no_preprocessing_statements(self,tmpdir): assert(process.returncode == 0) compile_process = sp.Popen([compile_executable], stdout=sp.PIPE, stderr=sp.PIPE) - c_out,c_err = compile_process.communicate() + c_out, c_err = compile_process.communicate() assert(p_out == c_out) From 5d112af39659c70775ebaeca91b44009a452c4f0 Mon Sep 17 00:00:00 2001 From: Jake Fenton Date: Tue, 13 Feb 2018 10:43:43 -0500 Subject: [PATCH 7/7] the littlest typo --- docs/_posts/2018-01-30-building-integration-tests.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/_posts/2018-01-30-building-integration-tests.md b/docs/_posts/2018-01-30-building-integration-tests.md index 7d6877d..8fdef59 100644 --- a/docs/_posts/2018-01-30-building-integration-tests.md +++ b/docs/_posts/2018-01-30-building-integration-tests.md @@ -22,7 +22,7 @@ Only in the GNU compiler, you can actually avoid changing the file name with usi Based on a global variable 'CXX_FLAG' which I'm hoping in the future to read the environment variable for it when it's being ran for either G++ or Clang. I don't see there being a huge need to support MSVC compilers, -but wht do I know. +but what do I know. ~~~ CXX_FLAG -Wall -std=c++17 -o output some_file.ii ~~~ @@ -40,4 +40,4 @@ A few things that trouble me about my current stream of execution moving to prod ###GPU Support As stated above, I really only thought about using Pepper with clang or g++ but applying NVICC support would be cool. - Well maybe just, modularized enough to support any C++ compiler. \ No newline at end of file + Well maybe just, modularized enough to support any C++ compiler.