From 61d824803c4c1a41f77a47e44949ebc93e224b37 Mon Sep 17 00:00:00 2001 From: still-flow <46608177+still-flow@users.noreply.github.com> Date: Thu, 28 May 2020 13:29:03 +0300 Subject: [PATCH 1/8] add simple concurrent compilation using something similar to round-robin scheduling --- Makefile.simple | 2 +- mymake.cpp | 47 +++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/Makefile.simple b/Makefile.simple index 79c75273..d8debc78 100644 --- a/Makefile.simple +++ b/Makefile.simple @@ -162,7 +162,7 @@ savepng$(OBJ_EXTENSION): savepng.cpp $(CXX) -O2 $(CXXFLAGS) -c savepng.cpp -o $@ mymake$(EXE_EXTENSION): mymake.cpp - $(CXX) -O2 $(CXXFLAGS) mymake.cpp -o $@ + $(CXX) -O2 $(CXXFLAGS) -lpthread mymake.cpp -o $@ emscripten: hyper.html diff --git a/mymake.cpp b/mymake.cpp index 455b36be..176e665c 100644 --- a/mymake.cpp +++ b/mymake.cpp @@ -14,6 +14,8 @@ #include #include #include +#include +#include using namespace std; @@ -27,6 +29,8 @@ string compiler; string linker; string libs; +int batch_size = 1; + void set_linux() { preprocessor = "g++ -E"; compiler = "g++ -Wall -Wextra -Wno-maybe-uninitialized -Wno-unused-parameter -Wno-implicit-fallthrough -rdynamic -fdiagnostics-color=always -c"; @@ -129,6 +133,10 @@ int main(int argc, char **argv) { standard = s; else if(s.substr(0, 2) == "-l") linker += " " + s; + else if(s.substr(0, 2) == "-j") + if (s.length() == 2 || stoi(s.substr(2)) < 1) + batch_size = thread::hardware_concurrency() + 1; + else batch_size = stoi(s.substr(2)); else if(s == "-I") { opts += " " + s + " " + argv[i+1]; i++; @@ -203,10 +211,10 @@ int main(int argc, char **argv) { } string allobj = " " + obj_dir + "/hyper.o"; - + int id = 0; + vector> tasks; for(string m: modules) { - id++; string src = m + ".cpp"; string m2 = m; for(char& c: m2) if(c == '/') c = '_'; @@ -218,14 +226,45 @@ int main(int argc, char **argv) { } time_t obj_time = get_file_time(obj); if(src_time > obj_time) { - printf("compiling %s... [%d/%d]\n", m.c_str(), id, int(modules.size())); - if(system(compiler + " " + opts + " " + src + " -o " + obj)) { printf("error\n"); exit(1); } + pair task(id, compiler + " " + opts + " " + src + " -o " + obj); + tasks.push_back(task); } else { printf("ok: %s\n", m.c_str()); } allobj += " "; allobj += obj; + id++; + } + + chrono::milliseconds quantum(25); + vector> workers(batch_size); + + int tasks_amt = tasks.size(); + int tasks_taken = 0, tasks_done = 0; + bool finished = tasks.empty(); + + while (!finished) + for (auto & worker : workers) { + check_lollygagging: + if (worker.valid()) { + if (worker.wait_for(quantum) != future_status::ready) continue; + else { + int res = worker.get(); + if (res) { printf("compilation error!\n"); exit(1); } + ++tasks_done; + goto check_lollygagging; + } + } + else if (tasks_taken < tasks_amt) { + auto task = tasks[tasks_taken]; + int mid = task.first; + string cmdline = task.second; + printf("compiling %s... [%d/%d]\n", modules[mid].c_str(), tasks_taken+1, tasks_amt); + worker = async(launch::async, (int (*)(string))system, cmdline); + ++tasks_taken; + } + else if (tasks_done == tasks_amt) { finished = true; break; } } printf("linking...\n"); From 9ef8b28129f49b487f07262e659fbf7592ccc9f5 Mon Sep 17 00:00:00 2001 From: still-flow <46608177+still-flow@users.noreply.github.com> Date: Thu, 28 May 2020 15:13:45 +0300 Subject: [PATCH 2/8] fix order of options to g++ in Makefile.simple --- Makefile.simple | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.simple b/Makefile.simple index d8debc78..eb948b68 100644 --- a/Makefile.simple +++ b/Makefile.simple @@ -162,7 +162,7 @@ savepng$(OBJ_EXTENSION): savepng.cpp $(CXX) -O2 $(CXXFLAGS) -c savepng.cpp -o $@ mymake$(EXE_EXTENSION): mymake.cpp - $(CXX) -O2 $(CXXFLAGS) -lpthread mymake.cpp -o $@ + $(CXX) -O2 $(CXXFLAGS) mymake.cpp -lpthread -o $@ emscripten: hyper.html From dcf81ee4681cf6b765b8728db2f0573ca3e2f4c3 Mon Sep 17 00:00:00 2001 From: still-flow <46608177+still-flow@users.noreply.github.com> Date: Thu, 28 May 2020 15:28:50 +0300 Subject: [PATCH 3/8] change default behavior to multi-tasking --- mymake.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mymake.cpp b/mymake.cpp index 176e665c..b8b7be8c 100644 --- a/mymake.cpp +++ b/mymake.cpp @@ -29,7 +29,7 @@ string compiler; string linker; string libs; -int batch_size = 1; +int batch_size = thread::hardware_concurrency() + 1; void set_linux() { preprocessor = "g++ -E"; @@ -134,9 +134,7 @@ int main(int argc, char **argv) { else if(s.substr(0, 2) == "-l") linker += " " + s; else if(s.substr(0, 2) == "-j") - if (s.length() == 2 || stoi(s.substr(2)) < 1) - batch_size = thread::hardware_concurrency() + 1; - else batch_size = stoi(s.substr(2)); + batch_size = stoi(s.substr(2)); else if(s == "-I") { opts += " " + s + " " + argv[i+1]; i++; From 4386f86efb88c065fd82b3bf42fc8ba8f88d3eb0 Mon Sep 17 00:00:00 2001 From: still-flow <46608177+still-flow@users.noreply.github.com> Date: Thu, 28 May 2020 18:32:29 +0300 Subject: [PATCH 4/8] add diagnostic message showing batch size --- mymake.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mymake.cpp b/mymake.cpp index b8b7be8c..2f349945 100644 --- a/mymake.cpp +++ b/mymake.cpp @@ -210,6 +210,8 @@ int main(int argc, char **argv) { string allobj = " " + obj_dir + "/hyper.o"; + printf("compiling modules using batch size of %d:\n", batch_size); + int id = 0; vector> tasks; for(string m: modules) { From b2a930f0f6ddd589f0903c8ecc087eed0741ff1e Mon Sep 17 00:00:00 2001 From: still-flow <46608177+still-flow@users.noreply.github.com> Date: Thu, 28 May 2020 20:00:33 +0300 Subject: [PATCH 5/8] -lpthread -> -pthread for mymake target --- Makefile.simple | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.simple b/Makefile.simple index eb948b68..02e951e3 100644 --- a/Makefile.simple +++ b/Makefile.simple @@ -162,7 +162,7 @@ savepng$(OBJ_EXTENSION): savepng.cpp $(CXX) -O2 $(CXXFLAGS) -c savepng.cpp -o $@ mymake$(EXE_EXTENSION): mymake.cpp - $(CXX) -O2 $(CXXFLAGS) mymake.cpp -lpthread -o $@ + $(CXX) -O2 $(CXXFLAGS) mymake.cpp -pthread -o $@ emscripten: hyper.html From 72008a7e850d4b54aa9aa71b6f00cfc17fd55bd4 Mon Sep 17 00:00:00 2001 From: still-flow <46608177+still-flow@users.noreply.github.com> Date: Thu, 28 May 2020 20:41:42 +0300 Subject: [PATCH 6/8] change tasks into pairs of ids and do_work functions --- mymake.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mymake.cpp b/mymake.cpp index 2f349945..5a823db8 100644 --- a/mymake.cpp +++ b/mymake.cpp @@ -16,6 +16,7 @@ #include #include #include +#include using namespace std; @@ -213,7 +214,7 @@ int main(int argc, char **argv) { printf("compiling modules using batch size of %d:\n", batch_size); int id = 0; - vector> tasks; + vector>> tasks; for(string m: modules) { string src = m + ".cpp"; string m2 = m; @@ -226,7 +227,8 @@ int main(int argc, char **argv) { } time_t obj_time = get_file_time(obj); if(src_time > obj_time) { - pair task(id, compiler + " " + opts + " " + src + " -o " + obj); + string cmdline = compiler + " " + opts + " " + src + " -o " + obj; + pair> task(id, [cmdline]() { return system(cmdline); }); tasks.push_back(task); } else { @@ -259,9 +261,9 @@ int main(int argc, char **argv) { else if (tasks_taken < tasks_amt) { auto task = tasks[tasks_taken]; int mid = task.first; - string cmdline = task.second; + function do_work = task.second; printf("compiling %s... [%d/%d]\n", modules[mid].c_str(), tasks_taken+1, tasks_amt); - worker = async(launch::async, (int (*)(string))system, cmdline); + worker = async(launch::async, do_work); ++tasks_taken; } else if (tasks_done == tasks_amt) { finished = true; break; } From d6de73a1dca57d8413a888379bcc700e234f261e Mon Sep 17 00:00:00 2001 From: still-flow <46608177+still-flow@users.noreply.github.com> Date: Sat, 30 May 2020 16:44:46 +0300 Subject: [PATCH 7/8] remove unnecessary goto which is a bit of a shame --- mymake.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mymake.cpp b/mymake.cpp index 5a823db8..89286895 100644 --- a/mymake.cpp +++ b/mymake.cpp @@ -248,17 +248,15 @@ int main(int argc, char **argv) { while (!finished) for (auto & worker : workers) { - check_lollygagging: if (worker.valid()) { if (worker.wait_for(quantum) != future_status::ready) continue; else { int res = worker.get(); if (res) { printf("compilation error!\n"); exit(1); } ++tasks_done; - goto check_lollygagging; } } - else if (tasks_taken < tasks_amt) { + if (tasks_taken < tasks_amt) { auto task = tasks[tasks_taken]; int mid = task.first; function do_work = task.second; From 31d836b2a9831be4016c99a458fb0c2482a0677e Mon Sep 17 00:00:00 2001 From: still-flow <46608177+still-flow@users.noreply.github.com> Date: Sat, 30 May 2020 17:17:51 +0300 Subject: [PATCH 8/8] restructure scheduling intervals so that there is just one pause for every sweep of the workforce, so that upper bound on idle time doesn't grow proportionally to the number of workers --- mymake.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mymake.cpp b/mymake.cpp index 89286895..4dbcdb97 100644 --- a/mymake.cpp +++ b/mymake.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -239,17 +240,17 @@ int main(int argc, char **argv) { id++; } - chrono::milliseconds quantum(25); + chrono::milliseconds quantum(40); vector> workers(batch_size); int tasks_amt = tasks.size(); int tasks_taken = 0, tasks_done = 0; bool finished = tasks.empty(); - while (!finished) + while (!finished) { for (auto & worker : workers) { if (worker.valid()) { - if (worker.wait_for(quantum) != future_status::ready) continue; + if (worker.wait_for(chrono::seconds(0)) != future_status::ready) continue; else { int res = worker.get(); if (res) { printf("compilation error!\n"); exit(1); } @@ -265,7 +266,7 @@ int main(int argc, char **argv) { ++tasks_taken; } else if (tasks_done == tasks_amt) { finished = true; break; } - } + } this_thread::sleep_for(quantum); } printf("linking...\n"); system(linker + allobj + libs);