started exploratory implementation of the runtime

.gitignore

@@ -1,2 +1,5 @@
-venv
+/venv
 __pycache__
+/dist
+/obj
+.ninja_log

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/customTargets.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="CLionExternalBuildManager">
+    <target id="5c1cb7ec-1aa4-4f6f-97c1-504abfc76bd3" name="main" defaultType="MAKE">
+      <configuration id="ea6404e0-1feb-4070-b1c1-4295409dace7" name="main">
+        <build type="MAKE">
+          <make targetName="runtime_lib/main.cpp" />
+        </build>
+        <clean type="MAKE">
+          <make targetName="clean" />
+        </clean>
+      </configuration>
+    </target>
+  </component>
+</project>

.idea/makefile.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="MakefileBuildTargetsManager">
+    <user-build-targets>
+      <build-target name="runtime_lib/main.cpp">
+        <build-configurations>
+          <build-configuration>
+            <make-targets>
+              <make-target>runtime_lib/main.cpp</make-target>
+            </make-targets>
+          </build-configuration>
+        </build-configurations>
+      </build-target>
+      <build-target name="clean">
+        <build-configurations>
+          <build-configuration>
+            <make-targets>
+              <make-target>clean</make-target>
+            </make-targets>
+          </build-configuration>
+        </build-configurations>
+      </build-target>
+    </user-build-targets>
+  </component>
+</project>

.idea/misc.xml

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="CidrRootsConfiguration">
+    <sourceRoots>
+      <file path="$PROJECT_DIR$/runtime_lib" />
+    </sourceRoots>
+    <excludeRoots>
+      <file path="$PROJECT_DIR$/dist" />
+      <file path="$PROJECT_DIR$/obj" />
+      <file path="$PROJECT_DIR$/venv" />
+    </excludeRoots>
+  </component>
+  <component name="ExternalStorageConfigurationManager" enabled="true" />
+  <component name="MakefileSettings">
+    <option name="linkedExternalProjectsSettings">
+      <MakefileProjectSettings>
+        <option name="buildOptions" value="--jobs 28" />
+        <option name="buildTarget" value="main" />
+        <option name="externalProjectPath" value="$PROJECT_DIR$" />
+        <option name="modules">
+          <set>
+            <option value="$PROJECT_DIR$" />
+          </set>
+        </option>
+        <option name="version" value="2" />
+      </MakefileProjectSettings>
+    </option>
+  </component>
+  <component name="MakefileWorkspace" PROJECT_DIR="$PROJECT_DIR$" />
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

build.ninja

@@ -0,0 +1,11 @@
+cflags = -Wall -Wextra -g
+print_flags = -fdiagnostics-color=always
+
+libraries = -lsodium
+
+rule cc
+    command = g++ $print_flags $cflags $libraries $in -o $out
+
+build dist/main: cc runtime_lib/main.cpp
+
+

runtime_lib/ProgramQueue.h

@@ -1,2 +1,117 @@
 #pragma once
 
+#include <functional>
+#include <unordered_map>
+#include <stack>
+#include <queue>
+#include <memory>
+#include <vector>
+#include <thread>
+#include <mutex> 
+#include <chrono>
+
+
+
+using call_frame_target = void*call_frame_target(void*, void*);
+
+struct packaged_call_frame {
+    void* arguments;
+}
+
+class package_of_work {
+    public:
+
+        package_of_work(int kind, call_frame_target target, void* scope_) : 
+            kind_id(kind), work_factor(1), target_function(target), scope(scope_), finalized(false)
+        {
+            queue = std::queue()
+        }
+
+        int kind_id;
+        float work_factor;
+        std::queue<packaged_call_frame> queue;
+
+        call_frame_target target_function;
+        void* scope;
+
+        bool finalized;
+}
+
+struct call_frame {
+    void* arguments;
+    void* scope;
+    call_frame_target target_function;
+    int id;
+}
+
+struct call_result {
+    int nothing;
+}
+
+
+class GlobalThreadPool {
+    public:
+        GlobalThreadPool();
+        
+        int create_package(void* scope, call_frame_target target_function);
+        void submit_to_package(int kind_id, void* arguments);
+        struct call_result join_package(int kind_id);
+        
+
+    private:
+        std::unordered_map<int, package_of_work> m_work_packages;
+        std::stack<package_of_work> m_work_package_stack;
+        std::mutex m_work_packages_lock;
+
+        std::queue<struct call_frame> m_calls;
+        std::mutex m_calls_lock;
+        
+        std::unordered_map<int, call_result> m_results;
+        std::mutex m_results_lock;
+
+        volatile int m_last_id;
+        std::mutex m_id_lock;
+
+        std::vector<std::thread> m_trheads;
+
+        void thread_worker();
+}
+
+void GlobalThreadPool::thread_worker() {
+    using namespace std::chrono_literals;
+    package_of_work* current = NULL;
+
+    while (true) {
+        if (m_work_package_stack.size > 0) {
+            // grab a reference to the first queue
+            current = m_work_package_stack.top;
+        }
+
+        // if not finalized, try again
+        if (!current->finalized && current->queue.size == 0) {
+            std::this_thread::sleep_for(100ms);
+            continue
+        }
+
+
+        
+    }
+
+}
+
+int GlobalThreadPool::create_package(void* scope, call_frame_target target_function) 
+{
+    this->id_lock.lock()
+    int kind_id = this->last_id++;
+    this->id_lock.unlock()
+
+    this->work_packages_lock.lock()
+    auto package = package_of_work(kind_id, target_function, scope);
+    this->work_packages[kind_id] = package;
+    this->work_package_stack.push(package);
+    this->work_packages_lock.unlock()
+
+    return kind_id;
+}
+
+

runtime_lib/main.cpp

@@ -0,0 +1,240 @@
+//
+// Created by anton on 6/22/22.
+//
+#include <iostream>
+#include <cassert>
+#include <functional>
+#include <unordered_map>
+#include <stack>
+#include <queue>
+#include <memory>
+#include <vector>
+#include <thread>
+#include <mutex>
+#include <chrono>
+#include <atomic>
+#include <random>
+#include <sodium.h>
+#include <unordered_set>
+#include <map>
+
+
+typedef void *call_frame_target(void *, void *);
+
+
+
+
+struct call_frame {
+    call_frame(void *argument, void *scope, call_frame_target *target, int id) :
+            arguments(argument), scope(scope),
+            target_function(target), id(id) {
+
+    }
+
+    void *arguments;
+    void *scope;
+    call_frame_target *target_function;
+    int id;
+};
+
+
+class GlobalThreadPool {
+public:
+    GlobalThreadPool() : m_queue(), m_queue_lock(), m_last_id(0) {
+        std::cout << "Init GlobalThreadPool" << std::endl;
+
+        m_thread_count = std::thread::hardware_concurrency();
+
+        m_threads = std::vector<std::thread>(m_thread_count);
+
+        for (size_t i = 0; i < m_thread_count; i++) {
+            m_threads[i] = std::thread([&] {
+                worker();
+            });
+        }
+    }
+
+    ~GlobalThreadPool() {
+        if (m_active) {
+            std::cout << "GlobalThreadPool destructor called before join!" << std::endl;
+            join();
+        }
+
+        assert(!m_active);
+    }
+
+    int insert(void *arguments, void *scope, call_frame_target *target_function) {
+        assert(("ThreadPool must be active when inserting! stop() or join() were called too early!", m_active));
+
+        int id = m_last_id++;
+        m_queue_lock.lock();
+        m_queue.emplace(arguments, scope, target_function, id);
+        m_queue_lock.unlock();
+
+        return id;
+    }
+
+    void stop() {
+        m_active = false;
+    }
+
+    void join() {
+        assert(m_active);
+
+        stop();
+
+        std::cout << "joining..." << std::endl;
+
+        for (size_t i = 0; i < m_thread_count; i++) {
+            if (m_threads[i].joinable()) {
+                m_threads[i].join();
+                std::cout << "thread " << i << " joined!" << std::endl;
+            } else {
+                std::cout << "thread " << i << " not joinable" << std::endl;
+            }
+        }
+        std::cout << "done!" << std::endl;
+
+
+        std::cout << "GlobalThreadPool finished!" << std::endl;
+    }
+
+    size_t thread_count() const { return m_thread_count; }
+
+private:
+
+    std::queue<struct call_frame> m_queue;
+    std::mutex m_queue_lock;
+    std::atomic<int> m_last_id;
+    std::vector<std::thread> m_threads;
+
+    std::atomic<bool> m_active = true;
+
+    size_t m_thread_count;
+
+
+    void worker() {
+        // std::cout << "Hello from worker #" << std::this_thread::get_id() << std::endl;
+        using namespace std::chrono_literals;
+
+        while (true) {
+            m_queue_lock.lock();
+
+            if (m_queue.empty()) {
+                m_queue_lock.unlock();
+                if (!m_active) {
+                    break;
+                }
+                std::this_thread::sleep_for(10ms);
+                continue;
+            }
+
+            struct call_frame item = m_queue.front();
+            m_queue.pop();
+            m_queue_lock.unlock();
+
+            try {
+                item.target_function(item.arguments, item.scope);
+            } catch (const std::exception& t) {
+                std::cerr << "Error in thread " << std::this_thread::get_id() << ": " << t.what() << "\n"
+                    << "  in work package #" << item.id << std::endl;
+            } catch (...) {
+                std::cerr << "Error in thread " << std::this_thread::get_id() << ": " << "unknown" << "\n"
+                          << "  in work package #" << item.id << std::endl;
+            }
+        }
+    }
+
+};
+
+
+void *task1(void *args, void *scope) {
+    //std::cout << "hello from task 1" << std::endl;
+
+    assert(("Scope can't be nullptr", scope != nullptr));
+
+    auto *print_lock = reinterpret_cast<std::mutex *>(scope);
+
+    using u64 = unsigned long long int;
+
+    u64 word = 0;
+    u64 len = 0;
+
+    long long int seed = 0;
+
+    std::random_device dev;
+
+    // FIXME: random_device entropy seems to be stuck at 0.
+    // currently falling back to libsodium, but that's not sustainable.
+    if (dev.entropy() == 0) {
+        randombytes_buf(&seed, sizeof(long long int));
+    } else {
+        seed = dev();
+    }
+
+    std::mt19937 rng(seed);
+    std::uniform_int_distribution<std::mt19937::result_type> next_number(0, 26);
+
+    std::map<u64, u64> lengths;
+    //std::unordered_map<u64, u64> words;
+
+    for (u64 i = 0; i < 10000000; i++) {
+        unsigned long num = next_number(rng);
+
+        if (num == 0) {
+            lengths[len]++;
+            //words[word]++;
+            word = 0;
+            len = 0;
+            continue;
+        }
+
+        word *= 26;
+        word += num;
+        len++;
+    }
+
+    print_lock->lock();
+
+    std::cout << "Word lengths:\n";
+    for (auto& pair : lengths) {
+        if (pair.first > 20) {
+            break;
+        }
+        std::cout << pair.first << ": " << pair.second << "\n";
+    }
+
+    std::cout << "Most used words:\n";
+
+    print_lock->unlock();
+
+    return nullptr;
+}
+
+int main() {
+    sodium_init();
+
+    std::cout << "Hello, World!" << std::endl;
+
+    auto pool = GlobalThreadPool();
+    std::mutex stdout_lock;
+
+    std::cout << "Using " << pool.thread_count() << " workers" << std::endl;
+
+    for (int i = 0; i < 2; i++) {
+        pool.insert(nullptr, &stdout_lock, task1);
+    }
+
+    std::cout << pool.insert(nullptr, &stdout_lock, task1) << std::endl;
+
+    pool.join();
+
+    return 0;
+}
+
+
+
+
+
+
+