From c464c162ec87b38ceda924c50da8eadb0347fe75 Mon Sep 17 00:00:00 2001 From: Alexis Glass Date: Sat, 11 Nov 2023 10:51:44 +0100 Subject: [PATCH 01/10] Add wasm/ts support --- CMakeLists.txt | 28 +- djinni/ts/DjinniModule.ts | 19 + djinni/wasm/djinni_wasm.cpp | 222 ++++++ djinni/wasm/djinni_wasm.hpp | 657 ++++++++++++++++++ docs/developer-guide.md | 2 +- test-suite/CMakeLists.txt | 24 + test-suite/Djinni.cmake | 42 ++ test-suite/handwritten-src/ts/ArrayTest.ts | 36 + test-suite/handwritten-src/ts/AsyncTest.ts | 93 +++ .../handwritten-src/ts/ClientInterfaceTest.ts | 54 ++ .../handwritten-src/ts/ConstantsTest.ts | 21 + .../handwritten-src/ts/CppExceptionTest.ts | 69 ++ test-suite/handwritten-src/ts/DataTest.ts | 62 ++ test-suite/handwritten-src/ts/DurationTest.ts | 47 ++ test-suite/handwritten-src/ts/EnumTest.ts | 63 ++ .../handwritten-src/ts/MapRecordTest.ts | 56 ++ .../ts/NestedCollectionTest.ts | 33 + test-suite/handwritten-src/ts/OutcomeTest.ts | 42 ++ .../handwritten-src/ts/PrimitiveListTest.ts | 30 + .../handwritten-src/ts/PrimitivesTest.ts | 33 + test-suite/handwritten-src/ts/ProtoTest.ts | 66 ++ .../handwritten-src/ts/SetRecordTest.ts | 30 + test-suite/handwritten-src/ts/TokenTest.ts | 63 ++ test-suite/handwritten-src/ts/WcharTest.ts | 22 + test-suite/handwritten-src/ts/main.ts | 24 + test-suite/handwritten-src/ts/run.sh | 8 + test-suite/handwritten-src/ts/test.html | 22 + test-suite/handwritten-src/ts/testutils.ts | 137 ++++ test-suite/handwritten-src/ts/tsconfig.json | 11 + test/wasm_list.txt | 12 + 30 files changed, 2024 insertions(+), 4 deletions(-) create mode 100644 djinni/ts/DjinniModule.ts create mode 100644 djinni/wasm/djinni_wasm.cpp create mode 100644 djinni/wasm/djinni_wasm.hpp create mode 100644 test-suite/handwritten-src/ts/ArrayTest.ts create mode 100644 test-suite/handwritten-src/ts/AsyncTest.ts create mode 100644 test-suite/handwritten-src/ts/ClientInterfaceTest.ts create mode 100644 test-suite/handwritten-src/ts/ConstantsTest.ts create mode 100644 test-suite/handwritten-src/ts/CppExceptionTest.ts create mode 100644 test-suite/handwritten-src/ts/DataTest.ts create mode 100644 test-suite/handwritten-src/ts/DurationTest.ts create mode 100644 test-suite/handwritten-src/ts/EnumTest.ts create mode 100644 test-suite/handwritten-src/ts/MapRecordTest.ts create mode 100644 test-suite/handwritten-src/ts/NestedCollectionTest.ts create mode 100644 test-suite/handwritten-src/ts/OutcomeTest.ts create mode 100644 test-suite/handwritten-src/ts/PrimitiveListTest.ts create mode 100644 test-suite/handwritten-src/ts/PrimitivesTest.ts create mode 100644 test-suite/handwritten-src/ts/ProtoTest.ts create mode 100644 test-suite/handwritten-src/ts/SetRecordTest.ts create mode 100644 test-suite/handwritten-src/ts/TokenTest.ts create mode 100644 test-suite/handwritten-src/ts/WcharTest.ts create mode 100644 test-suite/handwritten-src/ts/main.ts create mode 100644 test-suite/handwritten-src/ts/run.sh create mode 100644 test-suite/handwritten-src/ts/test.html create mode 100644 test-suite/handwritten-src/ts/testutils.ts create mode 100644 test-suite/handwritten-src/ts/tsconfig.json create mode 100644 test/wasm_list.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index f6691c9..4d07bbe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,6 +48,11 @@ set(SRC_CPPCLI "djinni/cppcli/WrapperCache.cpp" ) +set(SRC_WASM + "djinni/wasm/djinni_wasm.hpp" + "djinni/wasm/djinni_wasm.cpp" +) + # set `DJINNI_LIBRARY_TYPE` to `STATIC` or `SHARED` to define the type of library. # If undefined, the type will be determined based on `BUILD_SHARED_LIBS` add_library(djinni_support_lib ${DJINNI_LIBRARY_TYPE} ${SRC_SHARED}) @@ -151,13 +156,14 @@ if(DJINNI_WITH_PYTHON) endif() option(DJINNI_WITH_CPPCLI "Include the C++/CLI support code in Djinni support library." OFF) +option(DJINNI_WITH_WASM "Include the WASM/TS support code in Djinni support library." OFF) if(DJINNI_WITH_CPPCLI) if(NOT MSVC) message(FATAL_ERROR "Enabling DJINNI_WITH_CPPCLI without MSVC is not supported") endif() - if(DJINNI_WITH_OBJC OR DJINNI_WITH_JNI OR DJINNI_WITH_PYTHON) + if(DJINNI_WITH_OBJC OR DJINNI_WITH_JNI OR DJINNI_WITH_PYTHON OR DJINNI_WITH_WASM) message(FATAL_ERROR "DJINNI_WITH_CPPCLI can not be used with other bindings enabled.") endif() @@ -180,8 +186,24 @@ if(DJINNI_WITH_CPPCLI) ) endif() -if(NOT (DJINNI_WITH_OBJC OR DJINNI_WITH_JNI OR DJINNI_WITH_PYTHON OR DJINNI_WITH_CPPCLI)) - message(FATAL_ERROR "At least one of DJINNI_WITH_OBJC or DJINNI_WITH_JNI or DJINNI_WITH_PYTHON or DJINNI_WITH_CPPCLI must be enabled.") +if(DJINNI_WITH_WASM) + + if(DJINNI_WITH_OBJC OR DJINNI_WITH_JNI OR DJINNI_WITH_PYTHON OR DJINNI_WITH_CPPCLI) + message(FATAL_ERROR "DJINNI_WITH_WASM can not be used with other bindings enabled.") + endif() + + target_sources(djinni_support_lib PRIVATE ${SRC_WASM}) + source_group("wasm", FILES ${SRC_WASM}) + install( + FILES + "djinni/wasm/djinni_wasm.hpp" + DESTINATION + ${CMAKE_INSTALL_INCLUDEDIR}/djinni/wasm + ) +endif() + +if(NOT (DJINNI_WITH_OBJC OR DJINNI_WITH_JNI OR DJINNI_WITH_PYTHON OR DJINNI_WITH_CPPCLI OR DJINNI_WITH_WASM)) + message(FATAL_ERROR "At least one of DJINNI_WITH_OBJC or DJINNI_WITH_JNI or DJINNI_WITH_PYTHON or DJINNI_WITH_CPPCLI or DJINNI_WITH_WASM must be enabled.") endif() option(DJINNI_BUILD_TESTING "Build tests" ON) diff --git a/djinni/ts/DjinniModule.ts b/djinni/ts/DjinniModule.ts new file mode 100644 index 0000000..d8b0f1d --- /dev/null +++ b/djinni/ts/DjinniModule.ts @@ -0,0 +1,19 @@ +/** + * Copyright 2021 Snap, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface DjinniModule { + allocateWasmBuffer(size: number): Uint8Array; +} diff --git a/djinni/wasm/djinni_wasm.cpp b/djinni/wasm/djinni_wasm.cpp new file mode 100644 index 0000000..0b05cb4 --- /dev/null +++ b/djinni/wasm/djinni_wasm.cpp @@ -0,0 +1,222 @@ +/** + * Copyright 2021 Snap, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "djinni_wasm.hpp" + +namespace djinni { + +Binary::CppType Binary::toCpp(const JsType& j) { + return PrimitiveArray, Binary>::toCpp(j); +} +Binary::JsType Binary::fromCpp(const CppType& c) { + static em::val arrayClass = em::val::global("Uint8Array"); + return PrimitiveArray, Binary>::fromCpp(c); +} +em::val Binary::getArrayClass() { + static em::val arrayClass = em::val::global("Uint8Array"); + return arrayClass; +} + +Date::CppType Date::toCpp(const JsType& j) { + auto nanosSinceEpoch = std::chrono::nanoseconds(static_cast(j.call("getTime").as() * 1'000'000)); + return CppType(std::chrono::duration_cast(nanosSinceEpoch)); +} +Date::JsType Date::fromCpp(const CppType& c) { + auto nanosSinceEpoch = std::chrono::duration_cast(c.time_since_epoch()); + static em::val dateType = em::val::global("Date"); + return dateType.new_(static_cast(nanosSinceEpoch.count()) / 1'000'000.0); +} + +JsProxyId nextId = 0; +std::unordered_map> jsProxyCache; +std::unordered_map cppProxyCache; +std::mutex jsProxyCacheMutex; +std::mutex cppProxyCacheMutex; + +JsProxyBase::JsProxyBase(const em::val& v) : _js(v), _id(_js["_djinni_js_proxy_id"].as()) { +} + +JsProxyBase::~JsProxyBase() { + std::lock_guard lk(jsProxyCacheMutex); + jsProxyCache.erase(_id); +} + +const em::val& JsProxyBase::_jsRef() const { + return _js; +} + +void JsProxyBase::checkError(const em::val& v) { + if (v.instanceof(em::val::global("Error"))) { + auto cppExceptionPtr = v["_djinni_cpp_exception_ptr"]; + if (!cppExceptionPtr.isUndefined()) { + std::exception_ptr* exptr = reinterpret_cast(cppExceptionPtr.as()); + std::rethrow_exception(*exptr); + } else { + throw JsException(v); + } + } +} + +void checkForNull(void* ptr, const char* context) { + if (!ptr) { + throw std::invalid_argument(std::string("nullptr is not allowed in ") + context); + } +} + +em::val getCppProxyFinalizerRegistry() { + static auto inst = em::val::module_property("cppProxyFinalizerRegistry"); + return inst; +} + +em::val getCppProxyClass() { + static auto inst = em::val::module_property("DjinniCppProxy"); + return inst; +} + +em::val getWasmMemoryBuffer() { + // When ALLOW_MEMORY_GROWTH is turned on, the WebAssembly.Memory object's underlying buffer changes as it grows, + // and the HEAP* views in the Emscripten module object get reset to new views over the bigger memory buffer. + // In this mode, capturing the heap here as a static variable is incorrect, and leads to runtime errors. + // https://github.com/emscripten-core/emscripten/blob/3.1.7/src/settings.js#L194-L207 is the growth setting + // https://github.com/emscripten-core/emscripten/blob/3.1.7/src/library.js#L127 is the call to reset the views after grow() + // https://github.com/emscripten-core/emscripten/blob/3.1.7/src/preamble.js#L269-L286 is where the views get reset + return em::val::module_property("HEAPU32")["buffer"]; +} + +em::val DataObject::createJsObject() { + static auto finalizerRegistry = em::val::module_property("directBufferFinalizerRegistry"); + static auto uint8ArrayClass = em::val::global("Uint8Array"); + em::val jsObj = uint8ArrayClass.new_(getWasmMemoryBuffer(), addr(), size()); + finalizerRegistry.call("register", jsObj, reinterpret_cast(this)); + return jsObj; +} + +static em::val allocateWasmBuffer(unsigned size) { + auto* dbuf = new GenericBuffer>(size); + return dbuf->createJsObject(); +} + +extern "C" EMSCRIPTEN_KEEPALIVE +void releaseWasmBuffer(unsigned addr) { + delete reinterpret_cast(addr); +} + +EM_JS(void, djinni_init_wasm, (), { + // console.log("djinni_init_wasm"); + Module.cppProxyFinalizerRegistry = new FinalizationRegistry(nativeRef => { + // console.log("finalizing cpp object @" + nativeRef); + nativeRef.nativeDestroy(); + nativeRef.delete(); + }); + + Module.directBufferFinalizerRegistry = new FinalizationRegistry(addr => { + Module._releaseWasmBuffer(addr); + }); + + class DjinniCppProxy { + constructor(nativeRef, methods) { + // console.log('new cpp proxy'); + this._djinni_native_ref = nativeRef; + let self = this; + methods.forEach(function(method) { + self[method] = function(...args) { + return nativeRef[method](...args); + } + }); + } + } + Module.DjinniCppProxy = DjinniCppProxy; + + class DjinniJsPromiseBuilder { + constructor(cppHandlerPtr) { + this.promise = new Promise((resolveFunc, rejectFunc) => { + Module.initCppResolveHandler(cppHandlerPtr, resolveFunc, rejectFunc); + }); + } + } + Module.DjinniJsPromiseBuilder = DjinniJsPromiseBuilder; + + Module.makeNativePromiseResolver = function(func, pNativePromise) { + return function(res) { + Module.resolveNativePromise(func, pNativePromise, res); + }; + }; + Module.makeNativePromiseRejecter = function(func, pNativePromise) { + return function(err) { + Module.rejectNativePromise(func, pNativePromise, err); + }; + }; + + Module.writeNativeMemory = function(src, nativePtr) { + var srcByteView = new Uint8Array(src.buffer, src.byteOffset, src.byteLength); + Module.HEAPU8.set(srcByteView, nativePtr); + }; + Module.readNativeMemory = function(cls, nativePtr, nativeSize) { + return new cls(Module.HEAPU8.buffer.slice(nativePtr, nativePtr + nativeSize)); + }; + + Module.protobuf = {}; + Module.registerProtobufLib = function(name, proto) { + Module.protobuf[name] = proto; + }; + + Module.callJsProxyMethod = function(obj, method, ...args) { + try { + return obj[method].apply(obj, args); + } catch (e) { + return e; + } + }; +}); + +EM_JS(void, djinni_register_name_in_ns, (const char* prefixedName, const char* namespacedName), { + prefixedName = readLatin1String(prefixedName); + namespacedName = readLatin1String(namespacedName); + let parts = namespacedName.split('.'); + let name = parts.pop(); + let ns = parts.reduce(function(path, part) { + if (!path.hasOwnProperty(part)) { path[part] = {}}; + return path[part] + }, Module); + ns[name] = Module[prefixedName]; +}); + +em::val djinni_native_exception_to_js(const std::exception& e) { + if (const auto* jsEx = dynamic_cast(&e)) { + return jsEx->cause(); + } else { + static std::exception_ptr exptr; + static auto ErrorClass = em::val::global("Error"); + auto error = ErrorClass.new_(std::string("C++: ") + e.what()); + exptr = std::current_exception(); + error.set("_djinni_cpp_exception_ptr", em::val(reinterpret_cast(&exptr))); + return error; + } +} + +void djinni_throw_native_exception(const std::exception& e) { + djinni_native_exception_to_js(e).throw_(); +} + +EMSCRIPTEN_BINDINGS(djinni_wasm) { + djinni_init_wasm(); + em::function("allocateWasmBuffer", &allocateWasmBuffer); + em::function("initCppResolveHandler", &CppResolveHandlerBase::initInstance); + em::function("resolveNativePromise", &CppResolveHandlerBase::resolveNativePromise); + em::function("rejectNativePromise", &CppResolveHandlerBase::rejectNativePromise); +} + +} diff --git a/djinni/wasm/djinni_wasm.hpp b/djinni/wasm/djinni_wasm.hpp new file mode 100644 index 0000000..8add002 --- /dev/null +++ b/djinni/wasm/djinni_wasm.hpp @@ -0,0 +1,657 @@ +/** + * Copyright 2021 Snap, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#ifdef __EMSCRIPTEN_PTHREADS__ +#include +#endif + +#include +#include +#include +#include +#include +#include + +namespace em = emscripten; + +namespace djinni { + +extern em::val getCppProxyFinalizerRegistry(); +extern em::val getCppProxyClass(); +extern em::val getWasmMemoryBuffer(); + +template +class InstanceTracker { +#ifdef DJINNI_WASM_TRACK_INSTANCES + static int& count() { + static int theCount = 0; + return theCount; + } +public: + InstanceTracker() { + std::cout << "++" << typeid(this).name() << " => " << ++count() << std::endl; + } + virtual ~InstanceTracker() { + std::cout << "--" << typeid(this).name() << " => " << --count() << std::endl; + } +#endif +}; + +template +class Primitive { +public: + using CppType = T; + using JsType = T; + + struct Boxed { + using JsType = em::val; + static CppType toCpp(JsType j) { + return j.as(); + } + static JsType fromCpp(CppType c) { + return JsType(c); + } + }; + + static CppType toCpp(const JsType& j) { + return j; + } + static JsType fromCpp(const CppType& c) { + return c; + } +}; + +using Bool = Primitive; +using I8 = Primitive; +using I16 = Primitive; +using I32 = Primitive; +using I64 = Primitive; +using F32 = Primitive; +using F64 = Primitive; +using String = Primitive; +using WString = Primitive; + +template +class WasmEnum { +public: + using CppType = T; + using JsType = int32_t; + + struct Boxed { + using JsType = em::val; + static CppType toCpp(JsType j) { + return static_cast(j.as()); + } + static JsType fromCpp(CppType c) { + return JsType(static_cast(c)); + } + }; + + static CppType toCpp(const JsType& j) { + return static_cast(j); + } + static JsType fromCpp(const CppType& c) { + return static_cast(c); + } +}; + +class Binary { +public: + using CppType = std::vector; + using JsType = em::val; + using Boxed = Binary; + + static CppType toCpp(const JsType& j); + static JsType fromCpp(const CppType& c); + + static em::val getArrayClass(); +}; + +class Date { +public: + using CppType = std::chrono::system_clock::time_point; + using JsType = em::val; + using Boxed = Date; + + static CppType toCpp(const JsType& j); + static JsType fromCpp(const CppType& c); +}; + +template