From 20325750ab639c19494c25fe61430e221e98eb02 Mon Sep 17 00:00:00 2001 From: Nick Gasson Date: Sun, 6 Nov 2022 20:36:05 +0000 Subject: [PATCH] Call registration function when AOT library is loaded --- src/cgen.c | 135 ++++++++++++++++++----- src/hash.c | 3 + src/jit/jit-core.c | 28 ++++- src/jit/jit-exits.c | 16 +++ src/jit/jit-llvm.c | 256 ++++++++++++++++++++++++++++++-------------- src/jit/jit-llvm.h | 2 +- src/jit/jit-optim.c | 3 + src/jit/jit-priv.h | 1 + src/symbols.txt | 2 + 9 files changed, 335 insertions(+), 111 deletions(-) diff --git a/src/cgen.c b/src/cgen.c index d2fe29d4..5d635c33 100644 --- a/src/cgen.c +++ b/src/cgen.c @@ -21,6 +21,7 @@ #include "diag.h" #include "hash.h" #include "jit/jit-ffi.h" +#include "jit/jit-llvm.h" #include "jit/jit.h" #include "lib.h" #include "opt.h" @@ -48,6 +49,25 @@ #include #include +#define CGEN_USE_JIT 0 + +typedef A(vcode_unit_t) unit_list_t; +typedef A(char *) obj_list_t; + +typedef struct { + unit_list_t units; + char *obj_path; + char *module_name; + unsigned index; + tree_t top; + cover_tagging_t *cover; +} cgen_job_t; + +static A(char *) link_args; +static A(char *) cleanup_files = AINIT; + +#if !CGEN_USE_JIT + #define DEBUG_METADATA_VERSION 3 #define CONST_REP_ARRAY_LIMIT 32 #define UNITS_PER_JOB 25 @@ -84,18 +104,6 @@ typedef enum { FUNC_ATTR_DLLEXPORT, } func_attr_t; -typedef A(vcode_unit_t) unit_list_t; -typedef A(char *) obj_list_t; - -typedef struct { - unit_list_t units; - char *obj_path; - char *module_name; - unsigned index; - tree_t top; - cover_tagging_t *cover; -} cgen_job_t; - typedef A(LLVMValueRef) llvm_value_list_t; static __thread LLVMModuleRef module = NULL; @@ -105,9 +113,6 @@ static __thread shash_t *string_pool = NULL; static __thread A(LLVMMetadataRef) debug_scopes; static __thread llvm_value_list_t ctors; -static A(char *) link_args; -static A(char *) cleanup_files = AINIT; - static LLVMValueRef cgen_support_fn(const char *name); static LLVMTypeRef cgen_state_type(vcode_unit_t unit); static void cgen_async_work(void *context, void *arg); @@ -4466,6 +4471,7 @@ static void cgen_module_debug_info(LLVMMetadataRef cu) cgen_push_debug_scope(mod); } +#endif static void cgen_find_children(vcode_unit_t root, unit_list_t *units) { @@ -4564,6 +4570,7 @@ static void cgen_find_units(vcode_unit_t root, unit_list_t *units) cgen_find_dependencies(units->items[i], units); } +#if !CGEN_USE_JIT static void cgen_partition_jobs(unit_list_t *units, workq_t *wq, const char *base_name, int units_per_job, tree_t top, cover_tagging_t *cover, @@ -5249,16 +5256,6 @@ static void cgen_abi_version(void) #endif } -static void cgen_link_arg(const char *fmt, ...) -{ - va_list ap; - va_start(ap, fmt); - char *buf = xvasprintf(fmt, ap); - va_end(ap); - - APUSH(link_args, buf); -} - static void cgen_native(LLVMTargetMachineRef tm_ref, char *obj_path) { char *error; @@ -5287,6 +5284,7 @@ static void cgen_native(LLVMTargetMachineRef tm_ref, char *obj_path) fatal("Failed to write bitcode to file"); #endif } +#endif static void cleanup_temp_dll(void) { @@ -5300,6 +5298,16 @@ static void cleanup_temp_dll(void) ACLEAR(cleanup_files); } +static void cgen_link_arg(const char *fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + char *buf = xvasprintf(fmt, ap); + va_end(ap); + + APUSH(link_args, buf); +} + static void cgen_link(const char *module_name, char **objs, int nobjs) { #ifdef LINKER_PATH @@ -5376,6 +5384,7 @@ static void cgen_link(const char *module_name, char **objs, int nobjs) ACLEAR(link_args); } +#if !CGEN_USE_JIT static void cgen_global_ctors(void) { if (ctors.count == 0) @@ -5577,3 +5586,79 @@ void cgen(tree_t top, vcode_unit_t vcode, cover_tagging_t *cover) workq_free(wq); } + +#else + +typedef struct { + jit_t *jit; + llvm_obj_t *obj; + cover_tagging_t *cover; +} cgen_req_t; + +static void cgen_async_cb(void *context, void *arg) +{ + cgen_req_t *req = context; + + vcode_unit_t vu = arg; + vcode_select_unit(vu); + + LOCAL_TEXT_BUF tb = tb_new(); + tb_istr(tb, vcode_unit_name()); + + jit_handle_t handle = jit_lazy_compile(req->jit, vcode_unit_name()); + assert(handle != JIT_HANDLE_INVALID); + + llvm_aot_compile(req->obj, req->jit, handle); +} + +void cgen(tree_t top, vcode_unit_t vcode, cover_tagging_t *cover) +{ + ident_t name = tree_ident(top); + if (tree_kind(top) == T_PACK_BODY) + name = tree_ident(tree_primary(top)); + + unit_list_t units = AINIT; + cgen_find_units(vcode, &units); + + LLVMInitializeNativeTarget(); + LLVMInitializeNativeAsmPrinter(); + + if (!LLVMIsMultithreaded()) + fatal("LLVM was built without multithreaded support"); + + cgen_req_t req = { + .jit = jit_new(), + .cover = cover, + .obj = llvm_obj_new(istr(name)), + }; + + stop_workers(); // XXX + + workq_t *wq = workq_new(&req); + + for (int i = 0; i < units.count; i++) + workq_do(wq, cgen_async_cb, units.items[i]); + + workq_start(wq); + workq_drain(wq); + + llvm_obj_emit(req.obj, "out.o"); + req.obj = NULL; + + progress("code generation for %d units", units.count); + + obj_list_t objs = AINIT; + APUSH(objs, xstrdup("out.o")); + + cgen_link(istr(name), objs.items, objs.count); + + for (unsigned i = 0; i < objs.count; i++) + free(objs.items[i]); + ACLEAR(objs); + + ACLEAR(units); + jit_free(req.jit); + workq_free(wq); +} + +#endif diff --git a/src/hash.c b/src/hash.c index befd42b5..c77c796c 100644 --- a/src/hash.c +++ b/src/hash.c @@ -207,6 +207,9 @@ shash_t *shash_new(int size) void shash_free(shash_t *h) { + if (h == NULL) + return; + for (unsigned i = 0; i < h->size; i++) { if (h->keys[i] != NULL) free(h->keys[i]); diff --git a/src/jit/jit-core.c b/src/jit/jit-core.c index b0d7af10..6d173fa3 100644 --- a/src/jit/jit-core.c +++ b/src/jit/jit-core.c @@ -234,12 +234,34 @@ jit_handle_t jit_compile(jit_t *j, ident_t name) return handle; jit_func_t *f = jit_get_func(j, handle); - if (f->irbuf == NULL && f->symbol == NULL) + if (f->irbuf == NULL && f->symbol == NULL && f->entry == jit_interp) jit_irgen(f); return handle; } +void jit_register(jit_t *j, const char *name, jit_entry_fn_t fn) +{ + jit_func_t *f = hash_get(j->index, name); + if (f != NULL) + fatal_trace("attempt to register existing function %s", name); + + f = xcalloc(sizeof(jit_func_t)); + + f->name = ident_new(name); + f->unit = vcode_find_unit(f->name); + f->jit = j; + f->handle = j->funcs.count; + f->next_tier = j->tiers; + f->hotness = f->next_tier ? f->next_tier->threshold : 0; + f->entry = fn; + + if (f->unit) hash_put(j->index, f->unit, f); + hash_put(j->index, f->name, f); + + APUSH(j->funcs, f); +} + void *jit_link(jit_t *j, jit_handle_t handle) { if (handle == JIT_HANDLE_INVALID) @@ -547,7 +569,7 @@ static bool jit_try_vcall(jit_t *j, jit_func_t *f, jit_scalar_t *result, static void jit_unpack_args(jit_func_t *f, jit_scalar_t *args, va_list ap) { - if (f->symbol == NULL && f->irbuf == NULL) + if (f->symbol == NULL && f->irbuf == NULL && f->entry == jit_interp) jit_irgen(f); // Ensure FFI spec is set const int nargs = ffi_count_args(f->spec); @@ -617,7 +639,7 @@ bool jit_try_call_packed(jit_t *j, jit_handle_t handle, jit_scalar_t context, { jit_func_t *f = jit_get_func(j, handle); - if (f->symbol == NULL && f->irbuf == NULL) + if (f->symbol == NULL && f->irbuf == NULL && f->entry == jit_interp) jit_irgen(f); // Ensure FFI spec is set assert(f->spec != 0); diff --git a/src/jit/jit-exits.c b/src/jit/jit-exits.c index 083de6f8..92e90cf8 100644 --- a/src/jit/jit-exits.c +++ b/src/jit/jit-exits.c @@ -1472,3 +1472,19 @@ void __nvc_setup_toggle_cb(sig_shared_t *ss, int32_t* toggle_mask) { x_cover_setup_toggle_cb(ss, toggle_mask); } + +DLLEXPORT +void __nvc_register(const char *name, jit_entry_fn_t fn) +{ + printf("register! name=%s fn=%p\n", name, fn); + + jit_t *j = jit_thread_local()->jit; + jit_register(j, name, fn); +} + +DLLEXPORT +void __nvc_trampoline(jit_func_t *f, jit_anchor_t *caller, jit_scalar_t *args) +{ + printf("trampoline! %s\n", istr(f->name)); + (*f->entry)(f, caller, args); +} diff --git a/src/jit/jit-llvm.c b/src/jit/jit-llvm.c index bec215fa..b4b07849 100644 --- a/src/jit/jit-llvm.c +++ b/src/jit/jit-llvm.c @@ -16,16 +16,19 @@ // #include "util.h" +#include "hash.h" #include "ident.h" -#include "lib.h" #include "jit/jit-llvm.h" #include "jit/jit-priv.h" +#include "lib.h" #include "opt.h" +#include "rt/rt.h" #include "thread.h" #include #include #include +#include #include #include @@ -57,6 +60,8 @@ typedef enum { LLVM_ENTRY_FN, LLVM_ANCHOR, + LLVM_CTOR_FN, + LLVM_CTOR, LLVM_LAST_TYPE } llvm_type_t; @@ -101,6 +106,7 @@ typedef enum { LLVM_MSPACE_ALLOC, LLVM_DO_FFICALL, LLVM_TRAMPOLINE, + LLVM_REGISTER, LLVM_LAST_FN, } llvm_fn_t; @@ -117,8 +123,9 @@ typedef struct _llvm_obj { LLVMTypeRef types[LLVM_LAST_TYPE]; LLVMValueRef fns[LLVM_LAST_FN]; LLVMTypeRef fntypes[LLVM_LAST_FN]; + LLVMValueRef ctor; text_buf_t *textbuf; - bool aot; + shash_t *string_pool; } llvm_obj_t; typedef struct _cgen_block { @@ -215,13 +222,19 @@ static void llvm_register_types(llvm_obj_t *obj) obj->types[LLVM_PTR] = LLVMPointerType(obj->types[LLVM_INT8], 0); #endif - LLVMTypeRef atypes[] = { - obj->types[LLVM_PTR], // Function - obj->types[LLVM_PTR], // Anchor - obj->types[LLVM_PTR] // Arguments - }; - obj->types[LLVM_ENTRY_FN] = LLVMFunctionType(obj->types[LLVM_VOID], atypes, - ARRAY_LEN(atypes), false); + { + LLVMTypeRef atypes[] = { + obj->types[LLVM_PTR], // Function + obj->types[LLVM_PTR], // Anchor + obj->types[LLVM_PTR] // Arguments + }; + obj->types[LLVM_ENTRY_FN] = LLVMFunctionType(obj->types[LLVM_VOID], + atypes, ARRAY_LEN(atypes), + false); + } + + obj->types[LLVM_CTOR_FN] = LLVMFunctionType(obj->types[LLVM_VOID], + NULL, 0, false); { LLVMTypeRef fields[] = { @@ -244,6 +257,20 @@ static void llvm_register_types(llvm_obj_t *obj) ARRAY_LEN(fields), false); } + + { + LLVMTypeRef fields[] = { + obj->types[LLVM_INT32], +#ifdef LLVM_HAS_OPAQUE_POINTERS + obj->types[LLVM_PTR], +#else + LLVMPointerType(obj->types[LLVM_CTOR_FN], 0), +#endif + obj->types[LLVM_PTR], + }; + obj->types[LLVM_CTOR] = LLVMStructTypeInContext(obj->context, fields, + ARRAY_LEN(fields), false); + } } static void llvm_dump_module(LLVMModuleRef module, const char *tag) @@ -303,24 +330,22 @@ static void llvm_finalise(llvm_obj_t *obj) llvm_dump_module(obj->module, "final"); } -static LLVMTargetMachineRef llvm_target_machine(void) +static LLVMTargetMachineRef llvm_target_machine(LLVMRelocMode reloc, + LLVMCodeModel model) { - static __thread LLVMTargetMachineRef tm_ref = NULL; - if (tm_ref == NULL) { - char *def_triple = LLVMGetDefaultTargetTriple(); - char *error; - LLVMTargetRef target_ref; - if (LLVMGetTargetFromTriple(def_triple, &target_ref, &error)) - fatal("failed to get LLVM target for %s: %s", def_triple, error); + char *def_triple = LLVMGetDefaultTargetTriple(); + char *error; + LLVMTargetRef target_ref; + if (LLVMGetTargetFromTriple(def_triple, &target_ref, &error)) + fatal("failed to get LLVM target for %s: %s", def_triple, error); - tm_ref = LLVMCreateTargetMachine(target_ref, def_triple, "", "", - LLVMCodeGenLevelDefault, - LLVMRelocDefault, - LLVMCodeModelJITDefault); - LLVMDisposeMessage(def_triple); - } + LLVMTargetMachineRef tm = LLVMCreateTargetMachine(target_ref, def_triple, + "", "", + LLVMCodeGenLevelDefault, + reloc, model); + LLVMDisposeMessage(def_triple); - return tm_ref; + return tm; } static LLVMBasicBlockRef llvm_append_block(llvm_obj_t *obj, LLVMValueRef fn, @@ -329,21 +354,7 @@ static LLVMBasicBlockRef llvm_append_block(llvm_obj_t *obj, LLVMValueRef fn, return LLVMAppendBasicBlockInContext(obj->context, fn, name); } -static LLVMTypeRef cgen_get_type(llvm_obj_t *obj, llvm_type_t which) -{ - if (obj->types[which] != NULL) - return obj->types[which]; - - LLVMTypeRef type = NULL; - switch (which) { - default: - fatal_trace("cannot generate type %d", which); - } - - return (obj->types[which] = type); -} - -static LLVMValueRef cgen_add_fn(llvm_obj_t *obj, const char *name, +static LLVMValueRef llvm_add_fn(llvm_obj_t *obj, const char *name, LLVMTypeRef type) { LLVMValueRef fn = LLVMGetNamedFunction(obj->module, name); @@ -367,7 +378,7 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) { jit_size_t sz = which - LLVM_ADD_OVERFLOW_S8; LLVMTypeRef int_type = obj->types[LLVM_INT8 + sz]; - LLVMTypeRef pair_type = cgen_get_type(obj, LLVM_PAIR_I8_I1 + sz); + LLVMTypeRef pair_type = obj->types[LLVM_PAIR_I8_I1 + sz]; LLVMTypeRef args[] = { int_type, int_type }; obj->fntypes[which] = LLVMFunctionType(pair_type, args, ARRAY_LEN(args), false); @@ -378,7 +389,7 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) "llvm.sadd.with.overflow.i32", "llvm.sadd.with.overflow.i64" }; - fn = cgen_add_fn(obj, names[sz], obj->fntypes[which]); + fn = llvm_add_fn(obj, names[sz], obj->fntypes[which]); } break; @@ -389,7 +400,7 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) { jit_size_t sz = which - LLVM_ADD_OVERFLOW_U8; LLVMTypeRef int_type = obj->types[LLVM_INT8 + sz]; - LLVMTypeRef pair_type = cgen_get_type(obj, LLVM_PAIR_I8_I1 + sz); + LLVMTypeRef pair_type = obj->types[LLVM_PAIR_I8_I1 + sz]; LLVMTypeRef args[] = { int_type, int_type }; obj->fntypes[which] = LLVMFunctionType(pair_type, args, ARRAY_LEN(args), false); @@ -400,7 +411,7 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) "llvm.uadd.with.overflow.i32", "llvm.uadd.with.overflow.i64" }; - fn = cgen_add_fn(obj, names[sz], obj->fntypes[which]); + fn = llvm_add_fn(obj, names[sz], obj->fntypes[which]); } break; @@ -411,7 +422,7 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) { jit_size_t sz = which - LLVM_SUB_OVERFLOW_S8; LLVMTypeRef int_type = obj->types[LLVM_INT8 + sz]; - LLVMTypeRef pair_type = cgen_get_type(obj, LLVM_PAIR_I8_I1 + sz); + LLVMTypeRef pair_type = obj->types[LLVM_PAIR_I8_I1 + sz]; LLVMTypeRef args[] = { int_type, int_type }; obj->fntypes[which] = LLVMFunctionType(pair_type, args, ARRAY_LEN(args), false); @@ -422,7 +433,7 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) "llvm.ssub.with.overflow.i32", "llvm.ssub.with.overflow.i64" }; - fn = cgen_add_fn(obj, names[sz], obj->fntypes[which]); + fn = llvm_add_fn(obj, names[sz], obj->fntypes[which]); } break; @@ -433,7 +444,7 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) { jit_size_t sz = which - LLVM_SUB_OVERFLOW_U8; LLVMTypeRef int_type = obj->types[LLVM_INT8 + sz]; - LLVMTypeRef pair_type = cgen_get_type(obj, LLVM_PAIR_I8_I1 + sz); + LLVMTypeRef pair_type = obj->types[LLVM_PAIR_I8_I1 + sz]; LLVMTypeRef args[] = { int_type, int_type }; obj->fntypes[which] = LLVMFunctionType(pair_type, args, ARRAY_LEN(args), false); @@ -444,7 +455,7 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) "llvm.usub.with.overflow.i32", "llvm.usub.with.overflow.i64" }; - fn = cgen_add_fn(obj, names[sz], obj->fntypes[which]); + fn = llvm_add_fn(obj, names[sz], obj->fntypes[which]); } break; @@ -455,7 +466,7 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) { jit_size_t sz = which - LLVM_MUL_OVERFLOW_S8; LLVMTypeRef int_type = obj->types[LLVM_INT8 + sz]; - LLVMTypeRef pair_type = cgen_get_type(obj, LLVM_PAIR_I8_I1 + sz); + LLVMTypeRef pair_type = obj->types[LLVM_PAIR_I8_I1 + sz]; LLVMTypeRef args[] = { int_type, int_type }; obj->fntypes[which] = LLVMFunctionType(pair_type, args, ARRAY_LEN(args), false); @@ -466,7 +477,7 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) "llvm.smul.with.overflow.i32", "llvm.smul.with.overflow.i64" }; - fn = cgen_add_fn(obj, names[sz], obj->fntypes[which]); + fn = llvm_add_fn(obj, names[sz], obj->fntypes[which]); } break; @@ -477,7 +488,7 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) { jit_size_t sz = which - LLVM_MUL_OVERFLOW_U8; LLVMTypeRef int_type = obj->types[LLVM_INT8 + sz]; - LLVMTypeRef pair_type = cgen_get_type(obj, LLVM_PAIR_I8_I1 + sz); + LLVMTypeRef pair_type = obj->types[LLVM_PAIR_I8_I1 + sz]; LLVMTypeRef args[] = { int_type, int_type }; obj->fntypes[which] = LLVMFunctionType(pair_type, args, ARRAY_LEN(args), false); @@ -488,7 +499,7 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) "llvm.umul.with.overflow.i32", "llvm.umul.with.overflow.i64" }; - fn = cgen_add_fn(obj, names[sz], obj->fntypes[which]); + fn = llvm_add_fn(obj, names[sz], obj->fntypes[which]); } break; @@ -501,7 +512,7 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) obj->fntypes[which] = LLVMFunctionType(obj->types[LLVM_DOUBLE], args, ARRAY_LEN(args), false); - fn = cgen_add_fn(obj, "llvm.pow.f64", obj->fntypes[which]); + fn = llvm_add_fn(obj, "llvm.pow.f64", obj->fntypes[which]); } break; @@ -511,7 +522,7 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) obj->fntypes[which] = LLVMFunctionType(obj->types[LLVM_DOUBLE], args, ARRAY_LEN(args), false); - fn = cgen_add_fn(obj, "llvm.round.f64", obj->fntypes[which]); + fn = llvm_add_fn(obj, "llvm.round.f64", obj->fntypes[which]); } break; @@ -525,7 +536,7 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) obj->fntypes[which] = LLVMFunctionType(obj->types[LLVM_VOID], args, ARRAY_LEN(args), false); - fn = cgen_add_fn(obj, "__nvc_do_exit", obj->fntypes[which]); + fn = llvm_add_fn(obj, "__nvc_do_exit", obj->fntypes[which]); } break; @@ -539,7 +550,7 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) obj->fntypes[which] = LLVMFunctionType(obj->types[LLVM_VOID], args, ARRAY_LEN(args), false); - fn = cgen_add_fn(obj, "__nvc_do_fficall", obj->fntypes[which]); + fn = llvm_add_fn(obj, "__nvc_do_fficall", obj->fntypes[which]); } break; @@ -549,7 +560,7 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) obj->fntypes[which] = LLVMFunctionType(obj->types[LLVM_PTR], args, ARRAY_LEN(args), false); - fn = cgen_add_fn(obj, "__nvc_getpriv", obj->fntypes[which]); + fn = llvm_add_fn(obj, "__nvc_getpriv", obj->fntypes[which]); } break; @@ -562,7 +573,7 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) obj->fntypes[which] = LLVMFunctionType(obj->types[LLVM_VOID], args, ARRAY_LEN(args), false); - fn = cgen_add_fn(obj, "__nvc_putpriv", obj->fntypes[which]); + fn = llvm_add_fn(obj, "__nvc_putpriv", obj->fntypes[which]); } break; @@ -575,14 +586,26 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) obj->fntypes[which] = LLVMFunctionType(obj->types[LLVM_PTR], args, ARRAY_LEN(args), false); - fn = cgen_add_fn(obj, "__nvc_mspace_alloc", obj->fntypes[which]); + fn = llvm_add_fn(obj, "__nvc_mspace_alloc", obj->fntypes[which]); } break; case LLVM_TRAMPOLINE: { obj->fntypes[which] = obj->types[LLVM_ENTRY_FN]; - fn = cgen_add_fn(obj, "__nvc_trampoline", obj->fntypes[which]); + fn = llvm_add_fn(obj, "__nvc_trampoline", obj->fntypes[which]); + } + break; + + case LLVM_REGISTER: + { + LLVMTypeRef args[] = { + obj->types[LLVM_PTR], + obj->types[LLVM_PTR] + }; + obj->fntypes[which] = LLVMFunctionType(obj->types[LLVM_VOID], args, + ARRAY_LEN(args), false); + fn = llvm_add_fn(obj, "__nvc_register", obj->fntypes[which]); } break; @@ -601,6 +624,38 @@ static LLVMValueRef llvm_call_fn(llvm_obj_t *obj, llvm_fn_t which, args, count, ""); } +static LLVMValueRef llvm_const_string(llvm_obj_t *obj, const char *str) +{ + if (obj->string_pool == NULL) + obj->string_pool = shash_new(256); + + LLVMValueRef ref = shash_get(obj->string_pool, str); + if (ref == NULL) { + const size_t len = strlen(str); + LLVMValueRef init = + LLVMConstStringInContext(obj->context, str, len, false); + ref = LLVMAddGlobal(obj->module, + LLVMArrayType(obj->types[LLVM_INT8], len + 1), + "const_string"); + LLVMSetGlobalConstant(ref, true); + LLVMSetInitializer(ref, init); + LLVMSetLinkage(ref, LLVMPrivateLinkage); + LLVMSetUnnamedAddr(ref, true); + + shash_put(obj->string_pool, str, ref); + } + +#ifdef LLVM_HAS_OPAQUE_POINTERS + return ref; +#else + LLVMValueRef indexes[] = { + llvm_int32(obj, 0), + llvm_int32(obj, 0) + }; + return LLVMBuildGEP(obj->builder, ref, indexes, ARRAY_LEN(indexes), ""); +#endif +} + //////////////////////////////////////////////////////////////////////////////// // JIT IR to LLVM lowering @@ -688,7 +743,7 @@ static LLVMValueRef cgen_get_value(llvm_obj_t *obj, cgen_block_t *cgb, case JIT_VALUE_HANDLE: return llvm_int32(obj, value.exit); case JIT_ADDR_ABS: - assert(!obj->aot || value.int64 == 0); + assert(obj->ctor == NULL || value.int64 == 0); return llvm_ptr(obj, (void *)(intptr_t)value.int64); case JIT_VALUE_FOREIGN: return llvm_ptr(obj, (void *)(intptr_t)0xdeadbeef); @@ -1210,7 +1265,7 @@ static void cgen_op_call(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) jit_func_t *callee = jit_get_func(cgb->func->source->jit, ir->arg1.handle); LLVMValueRef fnptr = NULL; - if (obj->aot) + if (obj->ctor != NULL) fnptr = llvm_get_fn(obj, LLVM_TRAMPOLINE); else { const char *name = cgen_istr(obj, callee->name); @@ -1559,10 +1614,21 @@ static void cgen_frame_anchor(llvm_obj_t *obj, cgen_func_t *func) LLVMBuildStore(obj->builder, llvm_int32(obj, 0), irpos_ptr); } -static void cgen_module(llvm_obj_t *obj, cgen_func_t *func) +static void cgen_function(llvm_obj_t *obj, cgen_func_t *func) { func->llvmfn = LLVMAddFunction(obj->module, func->name, obj->types[LLVM_ENTRY_FN]); + LLVMSetLinkage(func->llvmfn, LLVMPrivateLinkage); + + if (obj->ctor != NULL) { + LLVMPositionBuilderAtEnd(obj->builder, LLVMGetLastBasicBlock(obj->ctor)); + + LLVMValueRef args[] = { + llvm_const_string(obj, func->name), + PTR(func->llvmfn) + }; + llvm_call_fn(obj, LLVM_REGISTER, args, ARRAY_LEN(args)); + } LLVMBasicBlockRef entry_bb = llvm_append_block(obj, func->llvmfn, "entry"); LLVMPositionBuilderAtEnd(obj->builder, entry_bb); @@ -1686,6 +1752,7 @@ typedef struct { LLVMOrcLLJITRef jit; LLVMOrcExecutionSessionRef session; LLVMOrcJITDylibRef dylib; + LLVMTargetMachineRef target; } lljit_state_t; static void *jit_llvm_init(void) @@ -1702,6 +1769,8 @@ static void *jit_llvm_init(void) state->session = LLVMOrcLLJITGetExecutionSession(state->jit); state->dylib = LLVMOrcLLJITGetMainJITDylib(state->jit); state->context = LLVMOrcCreateNewThreadSafeContext(); + state->target = llvm_target_machine(LLVMRelocDefault, + LLVMCodeModelJITDefault); const char prefix = LLVMOrcLLJITGetGlobalPrefix(state->jit); @@ -1728,7 +1797,7 @@ static void jit_llvm_cgen(jit_t *j, jit_handle_t handle, void *context) llvm_obj_t obj = { .context = LLVMOrcThreadSafeContextGetContext(state->context), - .target = llvm_target_machine(), + .target = state->target, .textbuf = tb_new(), }; @@ -1746,7 +1815,7 @@ static void jit_llvm_cgen(jit_t *j, jit_handle_t handle, void *context) .source = f, }; - cgen_module(&obj, &func); + cgen_function(&obj, &func); llvm_finalise(&obj); @@ -1805,17 +1874,13 @@ void jit_register_llvm_plugin(jit_t *j) llvm_obj_t *llvm_obj_new(const char *name) { - LLVMInitializeNativeTarget(); - LLVMInitializeNativeAsmPrinter(); - llvm_obj_t *obj = xcalloc(sizeof(llvm_obj_t)); obj->context = LLVMContextCreate(); obj->module = LLVMModuleCreateWithNameInContext(name, obj->context); obj->builder = LLVMCreateBuilderInContext(obj->context); - obj->target = llvm_target_machine(); + obj->target = llvm_target_machine(LLVMRelocPIC, LLVMCodeModelDefault); obj->textbuf = tb_new(); obj->data_ref = LLVMCreateTargetDataLayout(obj->target); - obj->aot = true; char *triple = LLVMGetTargetMachineTriple(obj->target); LLVMSetTarget(obj->module, triple); @@ -1825,6 +1890,35 @@ llvm_obj_t *llvm_obj_new(const char *name) llvm_register_types(obj); + obj->ctor = LLVMAddFunction(obj->module, "ctor", obj->types[LLVM_CTOR_FN]); + LLVMSetLinkage(obj->ctor, LLVMPrivateLinkage); + + llvm_append_block(obj, obj->ctor, "entry"); + + LLVMValueRef entry = LLVMGetUndef(obj->types[LLVM_CTOR]); + entry = LLVMBuildInsertValue(obj->builder, entry, + llvm_int32(obj, 65535), 0, ""); + entry = LLVMBuildInsertValue(obj->builder, entry, obj->ctor, 1, ""); + entry = LLVMBuildInsertValue(obj->builder, entry, + LLVMConstNull(obj->types[LLVM_PTR]), 2, ""); + + LLVMTypeRef array_type = LLVMArrayType(obj->types[LLVM_CTOR], 1); + LLVMValueRef global = + LLVMAddGlobal(obj->module, array_type, "llvm.global_ctors"); + LLVMSetLinkage(global, LLVMAppendingLinkage); + + LLVMValueRef ctors[] = { entry }; + LLVMValueRef array = LLVMConstArray(obj->types[LLVM_CTOR], ctors, 1); + LLVMSetInitializer(global, array); + + LLVMValueRef abi_version = + LLVMAddGlobal(obj->module, obj->types[LLVM_INT32], "__nvc_abi_version"); + LLVMSetInitializer(abi_version, llvm_int32(obj, RT_ABI_VERSION)); + LLVMSetGlobalConstant(abi_version, true); +#ifdef IMPLIB_REQUIRED + LLVMSetDLLStorageClass(abi_version, LLVMDLLExportStorageClass); +#endif + return obj; } @@ -1844,7 +1938,7 @@ void llvm_aot_compile(llvm_obj_t *obj, jit_t *j, jit_handle_t handle) .source = f, }; - cgen_module(obj, &func); + cgen_function(obj, &func); const uint64_t end_us = get_timestamp_us(); static __thread uint64_t slowest = 0; @@ -1855,28 +1949,26 @@ void llvm_aot_compile(llvm_obj_t *obj, jit_t *j, jit_handle_t handle) free(func.name); } -void llvm_obj_emit(llvm_obj_t *obj, const char *file) +void llvm_obj_emit(llvm_obj_t *obj, const char *path) { - llvm_finalise(obj); + LLVMPositionBuilderAtEnd(obj->builder, LLVMGetLastBasicBlock(obj->ctor)); + LLVMBuildRetVoid(obj->builder); - // TODO: emit to file + llvm_finalise(obj); - LLVMDumpModule(obj->module); + char *error; + if (LLVMTargetMachineEmitToFile(obj->target, obj->module, path, + LLVMObjectFile, &error)) + fatal("Failed to write object file: %s", error); LLVMDisposeTargetData(obj->data_ref); - obj->data_ref = NULL; - + LLVMDisposeTargetMachine(obj->target); LLVMDisposeBuilder(obj->builder); - obj->builder = NULL; - LLVMDisposeModule(obj->module); - obj->module = NULL; - LLVMContextDispose(obj->context); - obj->context = NULL; tb_free(obj->textbuf); - obj->textbuf = NULL; + shash_free(obj->string_pool); free(obj); } diff --git a/src/jit/jit-llvm.h b/src/jit/jit-llvm.h index 78d7201e..b5585620 100644 --- a/src/jit/jit-llvm.h +++ b/src/jit/jit-llvm.h @@ -29,6 +29,6 @@ typedef struct _llvm_obj llvm_obj_t; llvm_obj_t *llvm_obj_new(const char *name); void llvm_aot_compile(llvm_obj_t *obj, jit_t *j, jit_handle_t handle); -void llvm_obj_emit(llvm_obj_t *obj, const char *file); +void llvm_obj_emit(llvm_obj_t *obj, const char *path); #endif // _JIT_LLVM_H diff --git a/src/jit/jit-optim.c b/src/jit/jit-optim.c index 5549cc23..00d9a80b 100644 --- a/src/jit/jit-optim.c +++ b/src/jit/jit-optim.c @@ -217,6 +217,9 @@ void jit_free_cfg(jit_func_t *f) mask_free(&b->livein); mask_free(&b->liveout); mask_free(&b->varkill); + + if (b->in.max > 4) free(b->in.u.external); + if (b->out.max > 4) free(b->out.u.external); } free(f->cfg); diff --git a/src/jit/jit-priv.h b/src/jit/jit-priv.h index b45d9a5d..43fdcc84 100644 --- a/src/jit/jit-priv.h +++ b/src/jit/jit-priv.h @@ -294,6 +294,7 @@ bool jit_has_runtime(jit_t *j); int jit_backedge_limit(jit_t *j); void jit_tier_up(jit_func_t *f); jit_thread_local_t *jit_thread_local(void); +void jit_register(jit_t *j, const char *name, jit_entry_fn_t fn); jit_cfg_t *jit_get_cfg(jit_func_t *f); void jit_free_cfg(jit_func_t *f); diff --git a/src/symbols.txt b/src/symbols.txt index 0b5b1850..5e9e9771 100644 --- a/src/symbols.txt +++ b/src/symbols.txt @@ -55,11 +55,13 @@ __nvc_push_scope; __nvc_putpriv; __nvc_range_fail; + __nvc_register; __nvc_release; __nvc_report; __nvc_resolve_signal; __nvc_setup_toggle_cb; __nvc_tlab; + __nvc_trampoline; __nvc_unreachable; _assert_fail; _canon_value; -- 2.39.2