From 4f37ca160c638e6646d3a60d403b7b15eabdd49d Mon Sep 17 00:00:00 2001 From: Nick Gasson Date: Sat, 5 Nov 2022 21:11:14 +0000 Subject: [PATCH] Add ahead-of-time mode to LLVM JIT --- src/jit/jit-dump.c | 2 + src/jit/jit-ffi.c | 5 + src/jit/jit-ffi.h | 2 + src/jit/jit-interp.c | 2 + src/jit/jit-irgen.c | 20 +- src/jit/jit-llvm.c | 1458 ++++++++++++++++++++++-------------------- src/jit/jit-llvm.h | 6 +- src/jit/jit-priv.h | 16 +- 8 files changed, 804 insertions(+), 707 deletions(-) diff --git a/src/jit/jit-dump.c b/src/jit/jit-dump.c index 6432af46..bd397744 100644 --- a/src/jit/jit-dump.c +++ b/src/jit/jit-dump.c @@ -125,6 +125,8 @@ static int jit_dump_value(jit_dump_t *d, jit_value_t value) return printf("%s", jit_exit_name(value.exit)); case JIT_VALUE_LOC: return printf("<%s:%d>", loc_file_str(&value.loc), value.loc.first_line); + case JIT_VALUE_FOREIGN: + return printf("$%s", istr(ffi_get_sym(value.foreign))); case JIT_VALUE_INVALID: return printf("???"); } diff --git a/src/jit/jit-ffi.c b/src/jit/jit-ffi.c index a1188dde..06e14bb1 100644 --- a/src/jit/jit-ffi.c +++ b/src/jit/jit-ffi.c @@ -319,3 +319,8 @@ void *ffi_find_symbol(jit_dll_t *dll, const char *name) #endif } } + +ident_t ffi_get_sym(jit_foreign_t *ff) +{ + return ff->sym; +} diff --git a/src/jit/jit-ffi.h b/src/jit/jit-ffi.h index 547c3f7d..2a6a33c1 100644 --- a/src/jit/jit-ffi.h +++ b/src/jit/jit-ffi.h @@ -61,6 +61,8 @@ jit_foreign_t *jit_ffi_bind(ident_t sym, ffi_spec_t spec, void *ptr); jit_foreign_t *jit_ffi_get(ident_t sym); void jit_ffi_call(jit_foreign_t *ff, jit_scalar_t *args); +ident_t ffi_get_sym(jit_foreign_t *ff); + int ffi_count_args(ffi_spec_t spec); ffi_uarray_t ffi_wrap_str(char *buf, size_t len); size_t ffi_uarray_len(const ffi_uarray_t *u); diff --git a/src/jit/jit-interp.c b/src/jit/jit-interp.c index 4931ac9e..0365d3e8 100644 --- a/src/jit/jit-interp.c +++ b/src/jit/jit-interp.c @@ -168,6 +168,8 @@ static jit_scalar_t interp_get_value(jit_interp_t *state, jit_value_t value) return (jit_scalar_t){ .integer = value.label }; case JIT_VALUE_HANDLE: return (jit_scalar_t){ .integer = value.handle }; + case JIT_VALUE_FOREIGN: + return (jit_scalar_t){ .pointer = value.foreign }; default: interp_dump(state); fatal_trace("cannot handle value kind %d", value.kind); diff --git a/src/jit/jit-irgen.c b/src/jit/jit-irgen.c index 59a9e85d..bc4bf0d2 100644 --- a/src/jit/jit-irgen.c +++ b/src/jit/jit-irgen.c @@ -161,11 +161,6 @@ static jit_value_t jit_addr_from_value(jit_value_t value, int32_t disp) } } -static jit_value_t jit_addr_from_ptr(void *ptr) -{ - return (jit_value_t){ .kind = JIT_ADDR_ABS, .int64 = (intptr_t)ptr }; -} - static jit_value_t jit_value_from_label(irgen_label_t *l) { return (jit_value_t){ .kind = JIT_VALUE_LABEL, .label = l->label }; @@ -181,6 +176,11 @@ static jit_value_t jit_value_from_exit(jit_exit_t exit) return (jit_value_t){ .kind = JIT_VALUE_EXIT, .exit = exit }; } +static jit_value_t jit_value_from_foreign(jit_foreign_t *ff) +{ + return (jit_value_t){ .kind = JIT_VALUE_FOREIGN, .foreign = ff }; +} + static jit_ir_t *irgen_append(jit_func_t *f) { if (f->nirs == f->bufsz) { @@ -586,11 +586,11 @@ static jit_value_t macro_exp(jit_irgen_t *g, jit_value_t lhs, jit_value_t rhs) return jit_value_from_reg(r); } -static void macro_fficall(jit_irgen_t *g, jit_value_t addr) +static void macro_fficall(jit_irgen_t *g, jit_value_t func) { - assert(jit_value_is_addr(addr)); + assert(func.kind == JIT_VALUE_FOREIGN); irgen_emit_unary(g, MACRO_FFICALL, JIT_SZ_UNSPEC, JIT_CC_NONE, - JIT_REG_INVALID, addr); + JIT_REG_INVALID, func); } static jit_value_t macro_getpriv(jit_irgen_t *g, jit_handle_t handle) @@ -2020,9 +2020,7 @@ static void irgen_op_fcall(jit_irgen_t *g, int op) irgen_send_args(g, op, 0); jit_foreign_t *ff = irgen_ffi_for_call(g, op); - - jit_value_t addr = jit_addr_from_ptr(ff); - macro_fficall(g, addr); + macro_fficall(g, jit_value_from_foreign(ff)); vcode_reg_t result = vcode_get_result(op); if (result != VCODE_INVALID_REG) { diff --git a/src/jit/jit-llvm.c b/src/jit/jit-llvm.c index e6a10d2d..bec215fa 100644 --- a/src/jit/jit-llvm.c +++ b/src/jit/jit-llvm.c @@ -18,12 +18,11 @@ #include "util.h" #include "ident.h" #include "lib.h" +#include "jit/jit-llvm.h" #include "jit/jit-priv.h" #include "opt.h" #include "thread.h" -#ifdef LLVM_HAS_LLJIT - #include #include #include @@ -34,9 +33,12 @@ #include #include #include -#include #include +#ifdef LLVM_HAS_LLJIT +#include +#endif + typedef enum { LLVM_VOID, LLVM_PTR, @@ -98,44 +100,47 @@ typedef enum { LLVM_PUTPRIV, LLVM_MSPACE_ALLOC, LLVM_DO_FFICALL, + LLVM_TRAMPOLINE, LLVM_LAST_FN, } llvm_fn_t; -typedef struct { +typedef struct _cgen_func cgen_func_t; +typedef struct _cgen_block cgen_block_t; + +typedef struct _llvm_obj { + LLVMModuleRef module; + LLVMContextRef context; + LLVMTargetMachineRef target; + LLVMBuilderRef builder; + LLVMTargetDataRef data_ref; + LLVMTypeRef types[LLVM_LAST_TYPE]; + LLVMValueRef fns[LLVM_LAST_FN]; + LLVMTypeRef fntypes[LLVM_LAST_FN]; + text_buf_t *textbuf; + bool aot; +} llvm_obj_t; + +typedef struct _cgen_block { LLVMBasicBlockRef bbref; LLVMValueRef inflags; LLVMValueRef outflags; LLVMValueRef *inregs; LLVMValueRef *outregs; jit_block_t *source; + cgen_func_t *func; } cgen_block_t; -typedef struct { - LLVMModuleRef module; - LLVMContextRef context; - LLVMBuilderRef builder; - LLVMTargetMachineRef target; - LLVMValueRef llvmfn; - LLVMValueRef args; - LLVMValueRef frame; - LLVMValueRef anchor; - LLVMTypeRef types[LLVM_LAST_TYPE]; - LLVMValueRef fns[LLVM_LAST_FN]; - LLVMTypeRef fntypes[LLVM_LAST_FN]; - cgen_block_t *blocks; - jit_func_t *func; - jit_cfg_t *cfg; - char *name; - text_buf_t *textbuf; -} cgen_req_t; - -typedef struct { - LLVMOrcThreadSafeContextRef context; - LLVMOrcLLJITRef jit; - LLVMOrcExecutionSessionRef session; - LLVMOrcJITDylibRef dylib; -} lljit_state_t; +typedef struct _cgen_func { + LLVMValueRef llvmfn; + LLVMValueRef args; + LLVMValueRef frame; + LLVMValueRef anchor; + cgen_block_t *blocks; + jit_func_t *source; + jit_cfg_t *cfg; + char *name; +} cgen_func_t; #define LLVM_CHECK(op, ...) do { \ LLVMErrorRef error = op(__VA_ARGS__); \ @@ -149,78 +154,209 @@ typedef struct { #define PTR(x) x #else #define PTR(x) \ - LLVMBuildPointerCast(req->builder, (x), req->types[LLVM_PTR], "") + LLVMBuildPointerCast(obj->builder, (x), obj->types[LLVM_PTR], "") #endif -static LLVMValueRef llvm_int1(cgen_req_t *req, bool b) +//////////////////////////////////////////////////////////////////////////////// +// LLVM wrappers + +static LLVMValueRef llvm_int1(llvm_obj_t *obj, bool b) +{ + return LLVMConstInt(obj->types[LLVM_INT1], b, false); +} + +static LLVMValueRef llvm_int8(llvm_obj_t *obj, int8_t i) { - return LLVMConstInt(req->types[LLVM_INT1], b, false); + return LLVMConstInt(obj->types[LLVM_INT8], i, false); } -static LLVMValueRef llvm_int8(cgen_req_t *req, int8_t i) +static LLVMValueRef llvm_int32(llvm_obj_t *obj, int32_t i) { - return LLVMConstInt(req->types[LLVM_INT8], i, false); + return LLVMConstInt(obj->types[LLVM_INT32], i, false); } -static LLVMValueRef llvm_int32(cgen_req_t *req, int32_t i) +static LLVMValueRef llvm_int64(llvm_obj_t *obj, int64_t i) { - return LLVMConstInt(req->types[LLVM_INT32], i, false); + return LLVMConstInt(obj->types[LLVM_INT64], i, false); } -static LLVMValueRef llvm_int64(cgen_req_t *req, int64_t i) +static LLVMValueRef llvm_intptr(llvm_obj_t *obj, intptr_t i) { - return LLVMConstInt(req->types[LLVM_INT64], i, false); + return LLVMConstInt(obj->types[LLVM_INTPTR], i, false); } -static LLVMValueRef llvm_intptr(cgen_req_t *req, intptr_t i) +static LLVMValueRef llvm_ptr(llvm_obj_t *obj, void *ptr) { - return LLVMConstInt(req->types[LLVM_INTPTR], i, false); + return LLVMConstIntToPtr(llvm_intptr(obj, (intptr_t)ptr), + obj->types[LLVM_PTR]); } -static LLVMValueRef llvm_ptr(cgen_req_t *req, void *ptr) +static LLVMValueRef llvm_real(llvm_obj_t *obj, double r) { - return LLVMConstIntToPtr(llvm_intptr(req, (intptr_t)ptr), - req->types[LLVM_PTR]); + return LLVMConstReal(obj->types[LLVM_DOUBLE], r); +} + +static void llvm_register_types(llvm_obj_t *obj) +{ + obj->types[LLVM_VOID] = LLVMVoidTypeInContext(obj->context); + obj->types[LLVM_INT1] = LLVMInt1TypeInContext(obj->context); + obj->types[LLVM_INT8] = LLVMInt8TypeInContext(obj->context); + obj->types[LLVM_INT16] = LLVMInt16TypeInContext(obj->context); + obj->types[LLVM_INT32] = LLVMInt32TypeInContext(obj->context); + obj->types[LLVM_INT64] = LLVMInt64TypeInContext(obj->context); + obj->types[LLVM_DOUBLE] = LLVMDoubleTypeInContext(obj->context); + + obj->types[LLVM_INTPTR] = LLVMIntPtrTypeInContext(obj->context, + obj->data_ref); + +#ifdef LLVM_HAS_OPAQUE_POINTERS + obj->types[LLVM_PTR] = LLVMPointerTypeInContext(obj->context, 0); +#else + obj->types[LLVM_PTR] = LLVMPointerType(obj->types[LLVM_INT8], 0); +#endif + + LLVMTypeRef atypes[] = { + obj->types[LLVM_PTR], // Function + obj->types[LLVM_PTR], // Anchor + obj->types[LLVM_PTR] // Arguments + }; + obj->types[LLVM_ENTRY_FN] = LLVMFunctionType(obj->types[LLVM_VOID], atypes, + ARRAY_LEN(atypes), false); + + { + LLVMTypeRef fields[] = { + obj->types[LLVM_PTR], // Caller + obj->types[LLVM_PTR], // Function + obj->types[LLVM_INT32] // IR position + }; + obj->types[LLVM_ANCHOR] = LLVMStructTypeInContext(obj->context, fields, + ARRAY_LEN(fields), + false); + } + + { + LLVMTypeRef fields[] = { + obj->types[LLVM_INT32], + obj->types[LLVM_INT1] + }; + obj->types[LLVM_PAIR_I32_I1] = LLVMStructTypeInContext(obj->context, + fields, + ARRAY_LEN(fields), + false); + } } -static LLVMValueRef llvm_real(cgen_req_t *req, double r) +static void llvm_dump_module(LLVMModuleRef module, const char *tag) { - return LLVMConstReal(req->types[LLVM_DOUBLE], r); + size_t length; + const char *module_name = LLVMGetModuleIdentifier(module, &length); + + if (!opt_get_verbose(OPT_LLVM_VERBOSE, module_name)) + return; + + LOCAL_TEXT_BUF tb = tb_new(); + tb_printf(tb, "%s.%s.ll", module_name, tag); + + char *error; + if (LLVMPrintModuleToFile(module, tb_get(tb), &error)) + fatal("Failed to write LLVM IR file: %s", error); + + debugf("wrote LLVM IR for %s to %s", module_name, tb_get(tb)); } -static LLVMBasicBlockRef cgen_append_block(cgen_req_t *req, const char *name) +static void llvm_verify_module(LLVMModuleRef module) { - return LLVMAppendBasicBlockInContext(req->context, req->llvmfn, name); +#ifdef DEBUG + if (LLVMVerifyModule(module, LLVMPrintMessageAction, NULL)) { + size_t len; + const char *name = LLVMGetModuleIdentifier(module, &len); + fatal("LLVM verification failed for %s", name); + } +#endif } -static LLVMTypeRef cgen_get_type(cgen_req_t *req, llvm_type_t which) +static void llvm_optimise(LLVMModuleRef module) { - if (req->types[which] != NULL) - return req->types[which]; + LLVMPassManagerRef fpm = LLVMCreateFunctionPassManagerForModule(module); + + LLVMAddScalarReplAggregatesPass(fpm); + LLVMAddInstructionCombiningPass(fpm); + LLVMAddReassociatePass(fpm); + LLVMAddGVNPass(fpm); + LLVMAddCFGSimplificationPass(fpm); + + LLVMInitializeFunctionPassManager(fpm); + + for (LLVMValueRef fn = LLVMGetFirstFunction(module); + fn != NULL; fn = LLVMGetNextFunction(fn)) + LLVMRunFunctionPassManager(fpm, fn); + + LLVMFinalizeFunctionPassManager(fpm); + LLVMDisposePassManager(fpm); +} + +static void llvm_finalise(llvm_obj_t *obj) +{ + llvm_dump_module(obj->module, "initial"); + llvm_verify_module(obj->module); + llvm_optimise(obj->module); + llvm_dump_module(obj->module, "final"); +} + +static LLVMTargetMachineRef llvm_target_machine(void) +{ + static __thread LLVMTargetMachineRef tm_ref = NULL; + if (tm_ref == NULL) { + char *def_triple = LLVMGetDefaultTargetTriple(); + char *error; + LLVMTargetRef target_ref; + if (LLVMGetTargetFromTriple(def_triple, &target_ref, &error)) + fatal("failed to get LLVM target for %s: %s", def_triple, error); + + tm_ref = LLVMCreateTargetMachine(target_ref, def_triple, "", "", + LLVMCodeGenLevelDefault, + LLVMRelocDefault, + LLVMCodeModelJITDefault); + LLVMDisposeMessage(def_triple); + } + + return tm_ref; +} + +static LLVMBasicBlockRef llvm_append_block(llvm_obj_t *obj, LLVMValueRef fn, + const char *name) +{ + return LLVMAppendBasicBlockInContext(obj->context, fn, name); +} + +static LLVMTypeRef cgen_get_type(llvm_obj_t *obj, llvm_type_t which) +{ + if (obj->types[which] != NULL) + return obj->types[which]; LLVMTypeRef type = NULL; switch (which) { - case LLVM_PAIR_I32_I1: - { - LLVMTypeRef fields[] = { - req->types[LLVM_INT32], - req->types[LLVM_INT1] - }; - type = LLVMStructTypeInContext(req->context, fields, 2, false); - } - break; - default: fatal_trace("cannot generate type %d", which); } - return (req->types[which] = type); + return (obj->types[which] = type); } -static LLVMValueRef cgen_get_fn(cgen_req_t *req, llvm_fn_t which) +static LLVMValueRef cgen_add_fn(llvm_obj_t *obj, const char *name, + LLVMTypeRef type) { - if (req->fns[which] != NULL) - return req->fns[which]; + LLVMValueRef fn = LLVMGetNamedFunction(obj->module, name); + if (fn == NULL) + fn = LLVMAddFunction(obj->module, name, type); + + return fn; +} + +static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) +{ + if (obj->fns[which] != NULL) + return obj->fns[which]; LLVMValueRef fn = NULL; switch (which) { @@ -230,10 +366,10 @@ static LLVMValueRef cgen_get_fn(cgen_req_t *req, llvm_fn_t which) case LLVM_ADD_OVERFLOW_S64: { jit_size_t sz = which - LLVM_ADD_OVERFLOW_S8; - LLVMTypeRef int_type = req->types[LLVM_INT8 + sz]; - LLVMTypeRef pair_type = cgen_get_type(req, LLVM_PAIR_I8_I1 + sz); + LLVMTypeRef int_type = obj->types[LLVM_INT8 + sz]; + LLVMTypeRef pair_type = cgen_get_type(obj, LLVM_PAIR_I8_I1 + sz); LLVMTypeRef args[] = { int_type, int_type }; - req->fntypes[which] = LLVMFunctionType(pair_type, args, + obj->fntypes[which] = LLVMFunctionType(pair_type, args, ARRAY_LEN(args), false); static const char *names[] = { @@ -242,7 +378,7 @@ static LLVMValueRef cgen_get_fn(cgen_req_t *req, llvm_fn_t which) "llvm.sadd.with.overflow.i32", "llvm.sadd.with.overflow.i64" }; - fn = LLVMAddFunction(req->module, names[sz], req->fntypes[which]); + fn = cgen_add_fn(obj, names[sz], obj->fntypes[which]); } break; @@ -252,10 +388,10 @@ static LLVMValueRef cgen_get_fn(cgen_req_t *req, llvm_fn_t which) case LLVM_ADD_OVERFLOW_U64: { jit_size_t sz = which - LLVM_ADD_OVERFLOW_U8; - LLVMTypeRef int_type = req->types[LLVM_INT8 + sz]; - LLVMTypeRef pair_type = cgen_get_type(req, LLVM_PAIR_I8_I1 + sz); + LLVMTypeRef int_type = obj->types[LLVM_INT8 + sz]; + LLVMTypeRef pair_type = cgen_get_type(obj, LLVM_PAIR_I8_I1 + sz); LLVMTypeRef args[] = { int_type, int_type }; - req->fntypes[which] = LLVMFunctionType(pair_type, args, + obj->fntypes[which] = LLVMFunctionType(pair_type, args, ARRAY_LEN(args), false); static const char *names[] = { @@ -264,7 +400,7 @@ static LLVMValueRef cgen_get_fn(cgen_req_t *req, llvm_fn_t which) "llvm.uadd.with.overflow.i32", "llvm.uadd.with.overflow.i64" }; - fn = LLVMAddFunction(req->module, names[sz], req->fntypes[which]); + fn = cgen_add_fn(obj, names[sz], obj->fntypes[which]); } break; @@ -274,10 +410,10 @@ static LLVMValueRef cgen_get_fn(cgen_req_t *req, llvm_fn_t which) case LLVM_SUB_OVERFLOW_S64: { jit_size_t sz = which - LLVM_SUB_OVERFLOW_S8; - LLVMTypeRef int_type = req->types[LLVM_INT8 + sz]; - LLVMTypeRef pair_type = cgen_get_type(req, LLVM_PAIR_I8_I1 + sz); + LLVMTypeRef int_type = obj->types[LLVM_INT8 + sz]; + LLVMTypeRef pair_type = cgen_get_type(obj, LLVM_PAIR_I8_I1 + sz); LLVMTypeRef args[] = { int_type, int_type }; - req->fntypes[which] = LLVMFunctionType(pair_type, args, + obj->fntypes[which] = LLVMFunctionType(pair_type, args, ARRAY_LEN(args), false); static const char *names[] = { @@ -286,7 +422,7 @@ static LLVMValueRef cgen_get_fn(cgen_req_t *req, llvm_fn_t which) "llvm.ssub.with.overflow.i32", "llvm.ssub.with.overflow.i64" }; - fn = LLVMAddFunction(req->module, names[sz], req->fntypes[which]); + fn = cgen_add_fn(obj, names[sz], obj->fntypes[which]); } break; @@ -296,10 +432,10 @@ static LLVMValueRef cgen_get_fn(cgen_req_t *req, llvm_fn_t which) case LLVM_SUB_OVERFLOW_U64: { jit_size_t sz = which - LLVM_SUB_OVERFLOW_U8; - LLVMTypeRef int_type = req->types[LLVM_INT8 + sz]; - LLVMTypeRef pair_type = cgen_get_type(req, LLVM_PAIR_I8_I1 + sz); + LLVMTypeRef int_type = obj->types[LLVM_INT8 + sz]; + LLVMTypeRef pair_type = cgen_get_type(obj, LLVM_PAIR_I8_I1 + sz); LLVMTypeRef args[] = { int_type, int_type }; - req->fntypes[which] = LLVMFunctionType(pair_type, args, + obj->fntypes[which] = LLVMFunctionType(pair_type, args, ARRAY_LEN(args), false); static const char *names[] = { @@ -308,7 +444,7 @@ static LLVMValueRef cgen_get_fn(cgen_req_t *req, llvm_fn_t which) "llvm.usub.with.overflow.i32", "llvm.usub.with.overflow.i64" }; - fn = LLVMAddFunction(req->module, names[sz], req->fntypes[which]); + fn = cgen_add_fn(obj, names[sz], obj->fntypes[which]); } break; @@ -318,10 +454,10 @@ static LLVMValueRef cgen_get_fn(cgen_req_t *req, llvm_fn_t which) case LLVM_MUL_OVERFLOW_S64: { jit_size_t sz = which - LLVM_MUL_OVERFLOW_S8; - LLVMTypeRef int_type = req->types[LLVM_INT8 + sz]; - LLVMTypeRef pair_type = cgen_get_type(req, LLVM_PAIR_I8_I1 + sz); + LLVMTypeRef int_type = obj->types[LLVM_INT8 + sz]; + LLVMTypeRef pair_type = cgen_get_type(obj, LLVM_PAIR_I8_I1 + sz); LLVMTypeRef args[] = { int_type, int_type }; - req->fntypes[which] = LLVMFunctionType(pair_type, args, + obj->fntypes[which] = LLVMFunctionType(pair_type, args, ARRAY_LEN(args), false); static const char *names[] = { @@ -330,7 +466,7 @@ static LLVMValueRef cgen_get_fn(cgen_req_t *req, llvm_fn_t which) "llvm.smul.with.overflow.i32", "llvm.smul.with.overflow.i64" }; - fn = LLVMAddFunction(req->module, names[sz], req->fntypes[which]); + fn = cgen_add_fn(obj, names[sz], obj->fntypes[which]); } break; @@ -340,10 +476,10 @@ static LLVMValueRef cgen_get_fn(cgen_req_t *req, llvm_fn_t which) case LLVM_MUL_OVERFLOW_U64: { jit_size_t sz = which - LLVM_MUL_OVERFLOW_U8; - LLVMTypeRef int_type = req->types[LLVM_INT8 + sz]; - LLVMTypeRef pair_type = cgen_get_type(req, LLVM_PAIR_I8_I1 + sz); + LLVMTypeRef int_type = obj->types[LLVM_INT8 + sz]; + LLVMTypeRef pair_type = cgen_get_type(obj, LLVM_PAIR_I8_I1 + sz); LLVMTypeRef args[] = { int_type, int_type }; - req->fntypes[which] = LLVMFunctionType(pair_type, args, + obj->fntypes[which] = LLVMFunctionType(pair_type, args, ARRAY_LEN(args), false); static const char *names[] = { @@ -352,100 +488,101 @@ static LLVMValueRef cgen_get_fn(cgen_req_t *req, llvm_fn_t which) "llvm.umul.with.overflow.i32", "llvm.umul.with.overflow.i64" }; - fn = LLVMAddFunction(req->module, names[sz], req->fntypes[which]); + fn = cgen_add_fn(obj, names[sz], obj->fntypes[which]); } break; case LLVM_POW_F64: { LLVMTypeRef args[] = { - req->types[LLVM_DOUBLE], - req->types[LLVM_DOUBLE] + obj->types[LLVM_DOUBLE], + obj->types[LLVM_DOUBLE] }; - req->fntypes[which] = LLVMFunctionType(req->types[LLVM_DOUBLE], + obj->fntypes[which] = LLVMFunctionType(obj->types[LLVM_DOUBLE], args, ARRAY_LEN(args), false); - fn = LLVMAddFunction(req->module, "llvm.pow.f64", req->fntypes[which]); + fn = cgen_add_fn(obj, "llvm.pow.f64", obj->fntypes[which]); } break; case LLVM_ROUND_F64: { - LLVMTypeRef args[] = { req->types[LLVM_DOUBLE] }; - req->fntypes[which] = LLVMFunctionType(req->types[LLVM_DOUBLE], + LLVMTypeRef args[] = { obj->types[LLVM_DOUBLE] }; + obj->fntypes[which] = LLVMFunctionType(obj->types[LLVM_DOUBLE], args, ARRAY_LEN(args), false); - fn = LLVMAddFunction(req->module, "llvm.round.f64", - req->fntypes[which]); + fn = cgen_add_fn(obj, "llvm.round.f64", obj->fntypes[which]); } break; case LLVM_DO_EXIT: { LLVMTypeRef args[] = { - req->types[LLVM_INT32], - req->types[LLVM_PTR], - req->types[LLVM_PTR] + obj->types[LLVM_INT32], + obj->types[LLVM_PTR], + obj->types[LLVM_PTR] }; - req->fntypes[which] = LLVMFunctionType(req->types[LLVM_VOID], args, + obj->fntypes[which] = LLVMFunctionType(obj->types[LLVM_VOID], args, ARRAY_LEN(args), false); - fn = LLVMAddFunction(req->module, "__nvc_do_exit", - req->fntypes[which]); + fn = cgen_add_fn(obj, "__nvc_do_exit", obj->fntypes[which]); } break; case LLVM_DO_FFICALL: { LLVMTypeRef args[] = { - req->types[LLVM_PTR], - req->types[LLVM_PTR], - req->types[LLVM_PTR] + obj->types[LLVM_PTR], + obj->types[LLVM_PTR], + obj->types[LLVM_PTR] }; - req->fntypes[which] = LLVMFunctionType(req->types[LLVM_VOID], args, + obj->fntypes[which] = LLVMFunctionType(obj->types[LLVM_VOID], args, ARRAY_LEN(args), false); - fn = LLVMAddFunction(req->module, "__nvc_do_fficall", - req->fntypes[which]); + fn = cgen_add_fn(obj, "__nvc_do_fficall", obj->fntypes[which]); } break; case LLVM_GETPRIV: { - LLVMTypeRef args[] = { req->types[LLVM_INT32] }; - req->fntypes[which] = LLVMFunctionType(req->types[LLVM_PTR], args, + LLVMTypeRef args[] = { obj->types[LLVM_INT32] }; + obj->fntypes[which] = LLVMFunctionType(obj->types[LLVM_PTR], args, ARRAY_LEN(args), false); - fn = LLVMAddFunction(req->module, "__nvc_getpriv", - req->fntypes[which]); + fn = cgen_add_fn(obj, "__nvc_getpriv", obj->fntypes[which]); } break; case LLVM_PUTPRIV: { LLVMTypeRef args[] = { - req->types[LLVM_INT32], - req->types[LLVM_PTR] + obj->types[LLVM_INT32], + obj->types[LLVM_PTR] }; - req->fntypes[which] = LLVMFunctionType(req->types[LLVM_VOID], args, + obj->fntypes[which] = LLVMFunctionType(obj->types[LLVM_VOID], args, ARRAY_LEN(args), false); - fn = LLVMAddFunction(req->module, "__nvc_putpriv", - req->fntypes[which]); + fn = cgen_add_fn(obj, "__nvc_putpriv", obj->fntypes[which]); } break; case LLVM_MSPACE_ALLOC: { LLVMTypeRef args[] = { - req->types[LLVM_INT32], - req->types[LLVM_INT32] + obj->types[LLVM_INT32], + obj->types[LLVM_INT32] }; - req->fntypes[which] = LLVMFunctionType(req->types[LLVM_PTR], args, + obj->fntypes[which] = LLVMFunctionType(obj->types[LLVM_PTR], args, ARRAY_LEN(args), false); - fn = LLVMAddFunction(req->module, "__nvc_mspace_alloc", - req->fntypes[which]); + fn = cgen_add_fn(obj, "__nvc_mspace_alloc", obj->fntypes[which]); + } + break; + + case LLVM_TRAMPOLINE: + { + obj->fntypes[which] = obj->types[LLVM_ENTRY_FN]; + fn = cgen_add_fn(obj, "__nvc_trampoline", obj->fntypes[which]); } break; @@ -453,17 +590,20 @@ static LLVMValueRef cgen_get_fn(cgen_req_t *req, llvm_fn_t which) fatal_trace("cannot generate prototype for function %d", which); } - return (req->fns[which] = fn); + return (obj->fns[which] = fn); } -static LLVMValueRef cgen_call_fn(cgen_req_t *req, llvm_fn_t which, +static LLVMValueRef llvm_call_fn(llvm_obj_t *obj, llvm_fn_t which, LLVMValueRef *args, unsigned count) { - LLVMValueRef fn = cgen_get_fn(req, which); - return LLVMBuildCall2(req->builder, req->fntypes[which], fn, + LLVMValueRef fn = llvm_get_fn(obj, which); + return LLVMBuildCall2(obj->builder, obj->fntypes[which], fn, args, count, ""); } +//////////////////////////////////////////////////////////////////////////////// +// JIT IR to LLVM lowering + static const char *cgen_reg_name(jit_reg_t reg) { #ifdef DEBUG @@ -488,75 +628,85 @@ static const char *cgen_arg_name(int nth) #endif } -static const char *cgen_istr(cgen_req_t *req, ident_t id) +static const char *cgen_istr(llvm_obj_t *obj, ident_t id) { - tb_rewind(req->textbuf); - tb_istr(req->textbuf, id); - return tb_get(req->textbuf); + tb_rewind(obj->textbuf); + tb_istr(obj->textbuf, id); + return tb_get(obj->textbuf); } -static LLVMValueRef cgen_get_value(cgen_req_t *req, cgen_block_t *cgb, +__attribute__((noreturn)) +static void cgen_abort(cgen_block_t *cgb, jit_ir_t *ir, const char *fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + + char *text LOCAL = xvasprintf(fmt, ap); + jit_dump_with_mark(cgb->func->source, ir - cgb->func->source->irbuf, false); + fatal_trace("%s", text); + + va_end(ap); +} + +static LLVMValueRef cgen_get_value(llvm_obj_t *obj, cgen_block_t *cgb, jit_value_t value) { switch (value.kind) { case JIT_VALUE_REG: - assert(value.reg < req->func->nregs); + assert(value.reg < cgb->func->source->nregs); assert(cgb->outregs[value.reg] != NULL); return cgb->outregs[value.reg]; case JIT_VALUE_INT64: - return llvm_int64(req, value.int64); + return llvm_int64(obj, value.int64); case JIT_VALUE_DOUBLE: - return llvm_real(req, value.dval); + return llvm_real(obj, value.dval); case JIT_ADDR_FRAME: { - assert(value.int64 >= 0 && value.int64 < req->func->framesz); - LLVMValueRef indexes[] = { llvm_intptr(req, value.int64) }; - LLVMTypeRef byte_type = req->types[LLVM_INT8]; - return LLVMBuildInBoundsGEP2(req->builder, byte_type, - req->frame, indexes, + assert(value.int64 >= 0 && value.int64 < cgb->func->source->framesz); + LLVMValueRef indexes[] = { llvm_intptr(obj, value.int64) }; + LLVMTypeRef byte_type = obj->types[LLVM_INT8]; + return LLVMBuildInBoundsGEP2(obj->builder, byte_type, + cgb->func->frame, indexes, ARRAY_LEN(indexes), ""); } case JIT_ADDR_CPOOL: - assert(value.int64 >= 0 && value.int64 <= req->func->cpoolsz); - return llvm_ptr(req, req->func->cpool + value.int64); + assert(value.int64 >= 0 && value.int64 <= cgb->func->source->cpoolsz); + return llvm_ptr(obj, cgb->func->source->cpool + value.int64); case JIT_ADDR_REG: { - assert(value.reg < req->func->nregs); + assert(value.reg < cgb->func->source->nregs); LLVMValueRef ptr = cgb->outregs[value.reg]; if (value.disp != 0) { - LLVMValueRef disp = llvm_int64(req, value.disp); - ptr = LLVMBuildAdd(req->builder, ptr, disp, ""); + LLVMValueRef disp = llvm_int64(obj, value.disp); + ptr = LLVMBuildAdd(obj->builder, ptr, disp, ""); } return ptr; } - /* - case JIT_ADDR_ABS: - return (jit_scalar_t){ .pointer = (void *)(intptr_t)value.int64 }; - case JIT_VALUE_LABEL: - return (jit_scalar_t){ .integer = value.label }; - */ case JIT_VALUE_EXIT: case JIT_VALUE_HANDLE: - return llvm_int32(req, value.exit); + return llvm_int32(obj, value.exit); case JIT_ADDR_ABS: - return llvm_ptr(req, (void *)(intptr_t)value.int64); + assert(!obj->aot || value.int64 == 0); + return llvm_ptr(obj, (void *)(intptr_t)value.int64); + case JIT_VALUE_FOREIGN: + return llvm_ptr(obj, (void *)(intptr_t)0xdeadbeef); default: fatal_trace("cannot handle value kind %d", value.kind); } } -static LLVMValueRef cgen_coerce_value(cgen_req_t *req, cgen_block_t *cgb, +static LLVMValueRef cgen_coerce_value(llvm_obj_t *obj, cgen_block_t *cgb, jit_value_t value, llvm_type_t type) { - LLVMValueRef raw = cgen_get_value(req, cgb, value); + LLVMValueRef raw = cgen_get_value(obj, cgb, value); LLVMTypeRef lltype = LLVMTypeOf(raw); switch (type) { case LLVM_PTR: if (LLVMGetTypeKind(lltype) == LLVMIntegerTypeKind) - return LLVMBuildIntToPtr(req->builder, raw, req->types[LLVM_PTR], ""); + return LLVMBuildIntToPtr(obj->builder, raw, obj->types[LLVM_PTR], ""); else return raw; @@ -568,25 +718,25 @@ static LLVMValueRef cgen_coerce_value(cgen_req_t *req, cgen_block_t *cgb, case LLVM_INT1: switch (LLVMGetTypeKind(lltype)) { case LLVMPointerTypeKind: - return LLVMBuildIntToPtr(req->builder, raw, req->types[LLVM_PTR], ""); + return LLVMBuildIntToPtr(obj->builder, raw, obj->types[LLVM_PTR], ""); case LLVMIntegerTypeKind: { const int bits1 = LLVMGetIntTypeWidth(lltype); - const int bits2 = LLVMGetIntTypeWidth(req->types[type]); + const int bits2 = LLVMGetIntTypeWidth(obj->types[type]); if (bits2 == 1) { LLVMValueRef zero = LLVMConstInt(lltype, 0, false); - return LLVMBuildICmp(req->builder, LLVMIntNE, raw, zero, ""); + return LLVMBuildICmp(obj->builder, LLVMIntNE, raw, zero, ""); } else if (bits1 < bits2) - return LLVMBuildSExt(req->builder, raw, req->types[type], ""); + return LLVMBuildSExt(obj->builder, raw, obj->types[type], ""); else if (bits1 == bits2) return raw; else - return LLVMBuildTrunc(req->builder, raw, req->types[type], ""); + return LLVMBuildTrunc(obj->builder, raw, obj->types[type], ""); } case LLVMDoubleTypeKind: - return LLVMBuildBitCast(req->builder, raw, req->types[type], ""); + return LLVMBuildBitCast(obj->builder, raw, obj->types[type], ""); default: LLVMDumpType(lltype); fatal_trace("cannot coerce type to integer"); @@ -594,14 +744,14 @@ static LLVMValueRef cgen_coerce_value(cgen_req_t *req, cgen_block_t *cgb, break; case LLVM_DOUBLE: - return LLVMBuildBitCast(req->builder, raw, req->types[type], ""); + return LLVMBuildBitCast(obj->builder, raw, obj->types[type], ""); default: return raw; } } -static void cgen_sext_result(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir, +static void cgen_sext_result(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir, LLVMValueRef value) { LLVMTypeRef type = LLVMTypeOf(value); @@ -612,14 +762,14 @@ static void cgen_sext_result(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir, cgb->outregs[ir->result] = value; } else - cgb->outregs[ir->result] = LLVMBuildSExt(req->builder, value, - req->types[LLVM_INT64], + cgb->outregs[ir->result] = LLVMBuildSExt(obj->builder, value, + obj->types[LLVM_INT64], cgen_reg_name(ir->result)); break; case LLVMDoubleTypeKind: - cgb->outregs[ir->result] = LLVMBuildBitCast(req->builder, value, - req->types[LLVM_INT64], + cgb->outregs[ir->result] = LLVMBuildBitCast(obj->builder, value, + obj->types[LLVM_INT64], cgen_reg_name(ir->result)); break; @@ -629,7 +779,7 @@ static void cgen_sext_result(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir, } } -static void cgen_zext_result(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir, +static void cgen_zext_result(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir, LLVMValueRef value) { LLVMTypeRef type = LLVMTypeOf(value); @@ -640,8 +790,8 @@ static void cgen_zext_result(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir, cgb->outregs[ir->result] = value; } else - cgb->outregs[ir->result] = LLVMBuildZExt(req->builder, value, - req->types[LLVM_INT64], + cgb->outregs[ir->result] = LLVMBuildZExt(obj->builder, value, + obj->types[LLVM_INT64], cgen_reg_name(ir->result)); break; @@ -651,109 +801,109 @@ static void cgen_zext_result(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir, } } -static void cgen_sync_irpos(cgen_req_t *req, jit_ir_t *ir) +static void cgen_sync_irpos(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - const unsigned irpos = ir - req->func->irbuf; - LLVMValueRef irpos_ptr = LLVMBuildStructGEP2(req->builder, - req->types[LLVM_ANCHOR], - req->anchor, 2, ""); - LLVMBuildStore(req->builder, llvm_int32(req, irpos), irpos_ptr); + const unsigned irpos = ir - cgb->func->source->irbuf; + LLVMValueRef irpos_ptr = LLVMBuildStructGEP2(obj->builder, + obj->types[LLVM_ANCHOR], + cgb->func->anchor, 2, "irpos"); + LLVMBuildStore(obj->builder, llvm_int32(obj, irpos), irpos_ptr); } -static void cgen_op_recv(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_recv(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { assert(ir->arg1.kind == JIT_VALUE_INT64); const int nth = ir->arg1.int64; assert(nth < JIT_MAX_ARGS); - LLVMValueRef indexes[] = { llvm_int32(req, nth) }; - LLVMTypeRef int64_type = req->types[LLVM_INT64]; + LLVMValueRef indexes[] = { llvm_int32(obj, nth) }; + LLVMTypeRef int64_type = obj->types[LLVM_INT64]; #ifdef LLVM_HAS_OPAQUE_POINTERS - LLVMValueRef cast = req->args; + LLVMValueRef cast = cgb->func->args; #else LLVMTypeRef ptr_type = LLVMPointerType(int64_type, 0); LLVMValueRef cast = - LLVMBuildPointerCast(req->builder, req->args, ptr_type, ""); + LLVMBuildPointerCast(obj->builder, cgb->func->args, ptr_type, ""); #endif - LLVMValueRef ptr = LLVMBuildInBoundsGEP2(req->builder, int64_type, + LLVMValueRef ptr = LLVMBuildInBoundsGEP2(obj->builder, int64_type, cast, indexes, ARRAY_LEN(indexes), cgen_arg_name(nth)); - cgb->outregs[ir->result] = LLVMBuildLoad2(req->builder, int64_type, ptr, + cgb->outregs[ir->result] = LLVMBuildLoad2(obj->builder, int64_type, ptr, cgen_reg_name(ir->result)); } -static void cgen_op_send(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_send(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { assert(ir->arg1.kind == JIT_VALUE_INT64); const int nth = ir->arg1.int64; - LLVMValueRef value = cgen_get_value(req, cgb, ir->arg2); + LLVMValueRef value = cgen_get_value(obj, cgb, ir->arg2); assert(nth < JIT_MAX_ARGS); - LLVMValueRef indexes[] = { llvm_int32(req, nth) }; - LLVMTypeRef int64_type = req->types[LLVM_INT64]; + LLVMValueRef indexes[] = { llvm_int32(obj, nth) }; + LLVMTypeRef int64_type = obj->types[LLVM_INT64]; #ifdef LLVM_HAS_OPAQUE_POINTERS - LLVMValueRef args_cast = req->args; + LLVMValueRef args_cast = cgb->func->args; #else LLVMTypeRef args_ptr_type = LLVMPointerType(int64_type, 0); LLVMValueRef args_cast = - LLVMBuildPointerCast(req->builder, req->args, args_ptr_type, ""); + LLVMBuildPointerCast(obj->builder, cgb->func->args, args_ptr_type, ""); #endif - LLVMValueRef ptr = LLVMBuildInBoundsGEP2(req->builder, int64_type, + LLVMValueRef ptr = LLVMBuildInBoundsGEP2(obj->builder, int64_type, args_cast, indexes, ARRAY_LEN(indexes), cgen_arg_name(nth)); #ifdef LLVM_HAS_OPAQUE_POINTERS - LLVMBuildStore(req->builder, value, ptr); + LLVMBuildStore(obj->builder, value, ptr); #else LLVMTypeRef ptr_type = LLVMPointerType(LLVMTypeOf(value), 0); - LLVMValueRef cast = LLVMBuildPointerCast(req->builder, ptr, ptr_type, ""); - LLVMBuildStore(req->builder, value, cast); + LLVMValueRef cast = LLVMBuildPointerCast(obj->builder, ptr, ptr_type, ""); + LLVMBuildStore(obj->builder, value, cast); #endif } -static void cgen_op_store(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_store(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { llvm_type_t type = LLVM_INT8 + ir->size; - LLVMValueRef value = cgen_coerce_value(req, cgb, ir->arg1, type); - LLVMValueRef ptr = cgen_coerce_value(req, cgb, ir->arg2, LLVM_PTR); + LLVMValueRef value = cgen_coerce_value(obj, cgb, ir->arg1, type); + LLVMValueRef ptr = cgen_coerce_value(obj, cgb, ir->arg2, LLVM_PTR); #ifdef LLVM_HAS_OPAQUE_POINTERS - LLVMBuildStore(req->builder, value, ptr); + LLVMBuildStore(obj->builder, value, ptr); #else LLVMTypeRef ptr_type = LLVMPointerType(LLVMTypeOf(value), 0); - LLVMValueRef cast = LLVMBuildPointerCast(req->builder, ptr, ptr_type, ""); - LLVMBuildStore(req->builder, value, cast); + LLVMValueRef cast = LLVMBuildPointerCast(obj->builder, ptr, ptr_type, ""); + LLVMBuildStore(obj->builder, value, cast); #endif } -static void cgen_op_load(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_load(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { llvm_type_t type = LLVM_INT8 + ir->size; - LLVMValueRef ptr = cgen_coerce_value(req, cgb, ir->arg1, LLVM_PTR); + LLVMValueRef ptr = cgen_coerce_value(obj, cgb, ir->arg1, LLVM_PTR); #ifndef LLVM_HAS_OPAQUE_POINTERS - LLVMTypeRef ptr_type = LLVMPointerType(req->types[type], 0); - ptr = LLVMBuildPointerCast(req->builder, ptr, ptr_type, ""); + LLVMTypeRef ptr_type = LLVMPointerType(obj->types[type], 0); + ptr = LLVMBuildPointerCast(obj->builder, ptr, ptr_type, ""); #endif if (type == LLVM_INT64) - cgb->outregs[ir->result] = LLVMBuildLoad2(req->builder, req->types[type], + cgb->outregs[ir->result] = LLVMBuildLoad2(obj->builder, obj->types[type], ptr, cgen_reg_name(ir->result)); else { LLVMValueRef tmp = - LLVMBuildLoad2(req->builder, req->types[type], ptr, ""); + LLVMBuildLoad2(obj->builder, obj->types[type], ptr, ""); if (ir->op == J_ULOAD) - cgen_zext_result(req, cgb, ir, tmp); + cgen_zext_result(obj, cgb, ir, tmp); else - cgen_sext_result(req, cgb, ir, tmp); + cgen_sext_result(obj, cgb, ir, tmp); } } -static void cgen_op_add(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_add(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { llvm_fn_t fn = LLVM_LAST_FN; if (ir->cc == JIT_CC_O) @@ -763,29 +913,29 @@ static void cgen_op_add(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) if (fn != LLVM_LAST_FN) { llvm_type_t type = LLVM_INT8 + ir->size; - LLVMValueRef arg1 = cgen_coerce_value(req, cgb, ir->arg1, type); - LLVMValueRef arg2 = cgen_coerce_value(req, cgb, ir->arg2, type); + LLVMValueRef arg1 = cgen_coerce_value(obj, cgb, ir->arg1, type); + LLVMValueRef arg2 = cgen_coerce_value(obj, cgb, ir->arg2, type); LLVMValueRef args[] = { arg1, arg2 }; - LLVMValueRef pair = cgen_call_fn(req, fn, args, 2); + LLVMValueRef pair = llvm_call_fn(obj, fn, args, 2); - LLVMValueRef result = LLVMBuildExtractValue(req->builder, pair, 0, ""); - cgb->outflags = LLVMBuildExtractValue(req->builder, pair, 1, "FLAGS"); + LLVMValueRef result = LLVMBuildExtractValue(obj->builder, pair, 0, ""); + cgb->outflags = LLVMBuildExtractValue(obj->builder, pair, 1, "FLAGS"); if (ir->cc == JIT_CC_C) - cgen_zext_result(req, cgb, ir, result); + cgen_zext_result(obj, cgb, ir, result); else - cgen_sext_result(req, cgb, ir, result); + cgen_sext_result(obj, cgb, ir, result); } else { - LLVMValueRef arg1 = cgen_get_value(req, cgb, ir->arg1); - LLVMValueRef arg2 = cgen_get_value(req, cgb, ir->arg2); - cgb->outregs[ir->result] = LLVMBuildAdd(req->builder, arg1, arg2, + LLVMValueRef arg1 = cgen_get_value(obj, cgb, ir->arg1); + LLVMValueRef arg2 = cgen_get_value(obj, cgb, ir->arg2); + cgb->outregs[ir->result] = LLVMBuildAdd(obj->builder, arg1, arg2, cgen_reg_name(ir->result)); } } -static void cgen_op_sub(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_sub(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { llvm_fn_t fn = LLVM_LAST_FN; if (ir->cc == JIT_CC_O) @@ -795,29 +945,29 @@ static void cgen_op_sub(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) if (fn != LLVM_LAST_FN) { llvm_type_t type = LLVM_INT8 + ir->size; - LLVMValueRef arg1 = cgen_coerce_value(req, cgb, ir->arg1, type); - LLVMValueRef arg2 = cgen_coerce_value(req, cgb, ir->arg2, type); + LLVMValueRef arg1 = cgen_coerce_value(obj, cgb, ir->arg1, type); + LLVMValueRef arg2 = cgen_coerce_value(obj, cgb, ir->arg2, type); LLVMValueRef args[] = { arg1, arg2 }; - LLVMValueRef pair = cgen_call_fn(req, fn, args, 2); + LLVMValueRef pair = llvm_call_fn(obj, fn, args, 2); - LLVMValueRef result = LLVMBuildExtractValue(req->builder, pair, 0, ""); - cgb->outflags = LLVMBuildExtractValue(req->builder, pair, 1, "FLAGS"); + LLVMValueRef result = LLVMBuildExtractValue(obj->builder, pair, 0, ""); + cgb->outflags = LLVMBuildExtractValue(obj->builder, pair, 1, "FLAGS"); if (ir->cc == JIT_CC_C) - cgen_zext_result(req, cgb, ir, result); + cgen_zext_result(obj, cgb, ir, result); else - cgen_sext_result(req, cgb, ir, result); + cgen_sext_result(obj, cgb, ir, result); } else { - LLVMValueRef arg1 = cgen_get_value(req, cgb, ir->arg1); - LLVMValueRef arg2 = cgen_get_value(req, cgb, ir->arg2); - cgb->outregs[ir->result] = LLVMBuildSub(req->builder, arg1, arg2, + LLVMValueRef arg1 = cgen_get_value(obj, cgb, ir->arg1); + LLVMValueRef arg2 = cgen_get_value(obj, cgb, ir->arg2); + cgb->outregs[ir->result] = LLVMBuildSub(obj->builder, arg1, arg2, cgen_reg_name(ir->result)); } } -static void cgen_op_mul(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_mul(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { llvm_fn_t fn = LLVM_LAST_FN; if (ir->cc == JIT_CC_O) @@ -827,185 +977,180 @@ static void cgen_op_mul(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) if (fn != LLVM_LAST_FN) { llvm_type_t type = LLVM_INT8 + ir->size; - LLVMValueRef arg1 = cgen_coerce_value(req, cgb, ir->arg1, type); - LLVMValueRef arg2 = cgen_coerce_value(req, cgb, ir->arg2, type); + LLVMValueRef arg1 = cgen_coerce_value(obj, cgb, ir->arg1, type); + LLVMValueRef arg2 = cgen_coerce_value(obj, cgb, ir->arg2, type); LLVMValueRef args[] = { arg1, arg2 }; - LLVMValueRef pair = cgen_call_fn(req, fn, args, 2); + LLVMValueRef pair = llvm_call_fn(obj, fn, args, 2); - LLVMValueRef result = LLVMBuildExtractValue(req->builder, pair, 0, ""); - cgb->outflags = LLVMBuildExtractValue(req->builder, pair, 1, "FLAGS"); + LLVMValueRef result = LLVMBuildExtractValue(obj->builder, pair, 0, ""); + cgb->outflags = LLVMBuildExtractValue(obj->builder, pair, 1, "FLAGS"); if (ir->cc == JIT_CC_C) - cgen_zext_result(req, cgb, ir, result); + cgen_zext_result(obj, cgb, ir, result); else - cgen_sext_result(req, cgb, ir, result); + cgen_sext_result(obj, cgb, ir, result); } else { - LLVMValueRef arg1 = cgen_get_value(req, cgb, ir->arg1); - LLVMValueRef arg2 = cgen_get_value(req, cgb, ir->arg2); - cgb->outregs[ir->result] = LLVMBuildMul(req->builder, arg1, arg2, + LLVMValueRef arg1 = cgen_get_value(obj, cgb, ir->arg1); + LLVMValueRef arg2 = cgen_get_value(obj, cgb, ir->arg2); + cgb->outregs[ir->result] = LLVMBuildMul(obj->builder, arg1, arg2, cgen_reg_name(ir->result)); } } -static void cgen_op_div(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_div(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef arg1 = cgen_get_value(req, cgb, ir->arg1); - LLVMValueRef arg2 = cgen_get_value(req, cgb, ir->arg2); + LLVMValueRef arg1 = cgen_get_value(obj, cgb, ir->arg1); + LLVMValueRef arg2 = cgen_get_value(obj, cgb, ir->arg2); - cgb->outregs[ir->result] = LLVMBuildSDiv(req->builder, arg1, arg2, + cgb->outregs[ir->result] = LLVMBuildSDiv(obj->builder, arg1, arg2, cgen_reg_name(ir->result)); } -static void cgen_op_rem(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_rem(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef arg1 = cgen_get_value(req, cgb, ir->arg1); - LLVMValueRef arg2 = cgen_get_value(req, cgb, ir->arg2); + LLVMValueRef arg1 = cgen_get_value(obj, cgb, ir->arg1); + LLVMValueRef arg2 = cgen_get_value(obj, cgb, ir->arg2); - cgb->outregs[ir->result] = LLVMBuildSRem(req->builder, arg1, arg2, + cgb->outregs[ir->result] = LLVMBuildSRem(obj->builder, arg1, arg2, cgen_reg_name(ir->result)); } -static void cgen_op_fadd(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_fadd(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef arg1 = cgen_coerce_value(req, cgb, ir->arg1, LLVM_DOUBLE); - LLVMValueRef arg2 = cgen_coerce_value(req, cgb, ir->arg2, LLVM_DOUBLE); + LLVMValueRef arg1 = cgen_coerce_value(obj, cgb, ir->arg1, LLVM_DOUBLE); + LLVMValueRef arg2 = cgen_coerce_value(obj, cgb, ir->arg2, LLVM_DOUBLE); - LLVMValueRef real = LLVMBuildFAdd(req->builder, arg1, arg2, ""); - cgen_sext_result(req, cgb, ir, real); + LLVMValueRef real = LLVMBuildFAdd(obj->builder, arg1, arg2, ""); + cgen_sext_result(obj, cgb, ir, real); } -static void cgen_op_fsub(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_fsub(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef arg1 = cgen_coerce_value(req, cgb, ir->arg1, LLVM_DOUBLE); - LLVMValueRef arg2 = cgen_coerce_value(req, cgb, ir->arg2, LLVM_DOUBLE); + LLVMValueRef arg1 = cgen_coerce_value(obj, cgb, ir->arg1, LLVM_DOUBLE); + LLVMValueRef arg2 = cgen_coerce_value(obj, cgb, ir->arg2, LLVM_DOUBLE); - LLVMValueRef real = LLVMBuildFSub(req->builder, arg1, arg2, ""); - cgen_sext_result(req, cgb, ir, real); + LLVMValueRef real = LLVMBuildFSub(obj->builder, arg1, arg2, ""); + cgen_sext_result(obj, cgb, ir, real); } -static void cgen_op_fmul(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_fmul(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef arg1 = cgen_coerce_value(req, cgb, ir->arg1, LLVM_DOUBLE); - LLVMValueRef arg2 = cgen_coerce_value(req, cgb, ir->arg2, LLVM_DOUBLE); + LLVMValueRef arg1 = cgen_coerce_value(obj, cgb, ir->arg1, LLVM_DOUBLE); + LLVMValueRef arg2 = cgen_coerce_value(obj, cgb, ir->arg2, LLVM_DOUBLE); - LLVMValueRef real = LLVMBuildFMul(req->builder, arg1, arg2, ""); - cgen_sext_result(req, cgb, ir, real); + LLVMValueRef real = LLVMBuildFMul(obj->builder, arg1, arg2, ""); + cgen_sext_result(obj, cgb, ir, real); } -static void cgen_op_fdiv(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_fdiv(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef arg1 = cgen_coerce_value(req, cgb, ir->arg1, LLVM_DOUBLE); - LLVMValueRef arg2 = cgen_coerce_value(req, cgb, ir->arg2, LLVM_DOUBLE); + LLVMValueRef arg1 = cgen_coerce_value(obj, cgb, ir->arg1, LLVM_DOUBLE); + LLVMValueRef arg2 = cgen_coerce_value(obj, cgb, ir->arg2, LLVM_DOUBLE); - LLVMValueRef real = LLVMBuildFDiv(req->builder, arg1, arg2, ""); - cgen_sext_result(req, cgb, ir, real); + LLVMValueRef real = LLVMBuildFDiv(obj->builder, arg1, arg2, ""); + cgen_sext_result(obj, cgb, ir, real); } -static void cgen_op_fneg(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_fneg(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef arg1 = cgen_coerce_value(req, cgb, ir->arg1, LLVM_DOUBLE); + LLVMValueRef arg1 = cgen_coerce_value(obj, cgb, ir->arg1, LLVM_DOUBLE); - LLVMValueRef real = LLVMBuildFNeg(req->builder, arg1, ""); - cgen_sext_result(req, cgb, ir, real); + LLVMValueRef real = LLVMBuildFNeg(obj->builder, arg1, ""); + cgen_sext_result(obj, cgb, ir, real); } -static void cgen_op_fcvtns(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_fcvtns(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef arg1 = cgen_coerce_value(req, cgb, ir->arg1, LLVM_DOUBLE); + LLVMValueRef arg1 = cgen_coerce_value(obj, cgb, ir->arg1, LLVM_DOUBLE); - LLVMValueRef fn = cgen_get_fn(req, LLVM_ROUND_F64); LLVMValueRef args[] = { arg1 }; - LLVMValueRef rounded = LLVMBuildCall2(req->builder, - req->fntypes[LLVM_ROUND_F64], fn, - args, ARRAY_LEN(args), ""); + LLVMValueRef rounded = llvm_call_fn(obj, LLVM_ROUND_F64, args, 1); - cgb->outregs[ir->result] = LLVMBuildFPToSI(req->builder, rounded, - req->types[LLVM_INT64], + cgb->outregs[ir->result] = LLVMBuildFPToSI(obj->builder, rounded, + obj->types[LLVM_INT64], cgen_reg_name(ir->result)); } -static void cgen_op_scvtf(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_scvtf(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef arg1 = cgen_get_value(req, cgb, ir->arg1); + LLVMValueRef arg1 = cgen_get_value(obj, cgb, ir->arg1); - LLVMValueRef real = LLVMBuildSIToFP(req->builder, arg1, - req->types[LLVM_DOUBLE], ""); - cgen_sext_result(req, cgb, ir, real); + LLVMValueRef real = LLVMBuildSIToFP(obj->builder, arg1, + obj->types[LLVM_DOUBLE], ""); + cgen_sext_result(obj, cgb, ir, real); } -static void cgen_op_not(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_not(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef arg1 = cgen_coerce_value(req, cgb, ir->arg1, LLVM_INT1); - LLVMValueRef logical = LLVMBuildNot(req->builder, arg1, ""); - cgen_zext_result(req, cgb, ir, logical); + LLVMValueRef arg1 = cgen_coerce_value(obj, cgb, ir->arg1, LLVM_INT1); + LLVMValueRef logical = LLVMBuildNot(obj->builder, arg1, ""); + cgen_zext_result(obj, cgb, ir, logical); } -static void cgen_op_and(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_and(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef arg1 = cgen_coerce_value(req, cgb, ir->arg1, LLVM_INT1); - LLVMValueRef arg2 = cgen_coerce_value(req, cgb, ir->arg2, LLVM_INT1); + LLVMValueRef arg1 = cgen_coerce_value(obj, cgb, ir->arg1, LLVM_INT1); + LLVMValueRef arg2 = cgen_coerce_value(obj, cgb, ir->arg2, LLVM_INT1); - LLVMValueRef logical = LLVMBuildAnd(req->builder, arg1, arg2, ""); - cgen_zext_result(req, cgb, ir, logical); + LLVMValueRef logical = LLVMBuildAnd(obj->builder, arg1, arg2, ""); + cgen_zext_result(obj, cgb, ir, logical); } -static void cgen_op_or(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_or(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef arg1 = cgen_coerce_value(req, cgb, ir->arg1, LLVM_INT1); - LLVMValueRef arg2 = cgen_coerce_value(req, cgb, ir->arg2, LLVM_INT1); + LLVMValueRef arg1 = cgen_coerce_value(obj, cgb, ir->arg1, LLVM_INT1); + LLVMValueRef arg2 = cgen_coerce_value(obj, cgb, ir->arg2, LLVM_INT1); - LLVMValueRef logical = LLVMBuildOr(req->builder, arg1, arg2, ""); - cgen_zext_result(req, cgb, ir, logical); + LLVMValueRef logical = LLVMBuildOr(obj->builder, arg1, arg2, ""); + cgen_zext_result(obj, cgb, ir, logical); } -static void cgen_op_xor(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_xor(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef arg1 = cgen_coerce_value(req, cgb, ir->arg1, LLVM_INT1); - LLVMValueRef arg2 = cgen_coerce_value(req, cgb, ir->arg2, LLVM_INT1); + LLVMValueRef arg1 = cgen_coerce_value(obj, cgb, ir->arg1, LLVM_INT1); + LLVMValueRef arg2 = cgen_coerce_value(obj, cgb, ir->arg2, LLVM_INT1); - LLVMValueRef logical = LLVMBuildXor(req->builder, arg1, arg2, ""); - cgen_zext_result(req, cgb, ir, logical); + LLVMValueRef logical = LLVMBuildXor(obj->builder, arg1, arg2, ""); + cgen_zext_result(obj, cgb, ir, logical); } -static void cgen_op_ret(cgen_req_t *req, jit_ir_t *ir) +static void cgen_op_ret(llvm_obj_t *obj, jit_ir_t *ir) { - LLVMBuildRetVoid(req->builder); + LLVMBuildRetVoid(obj->builder); } -static void cgen_op_jump(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_jump(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { if (ir->cc == JIT_CC_NONE) { assert(cgb->source->out.count == 1); LLVMBasicBlockRef dest = - req->blocks[jit_get_edge(&(cgb->source->out), 0)].bbref; - LLVMBuildBr(req->builder, dest); + cgb->func->blocks[jit_get_edge(&(cgb->source->out), 0)].bbref; + LLVMBuildBr(obj->builder, dest); } else if (ir->cc == JIT_CC_T) { assert(cgb->source->out.count == 2); LLVMBasicBlockRef dest_t = - req->blocks[jit_get_edge(&(cgb->source->out), 1)].bbref; + cgb->func->blocks[jit_get_edge(&(cgb->source->out), 1)].bbref; LLVMBasicBlockRef dest_f = (cgb + 1)->bbref; - LLVMBuildCondBr(req->builder, cgb->outflags, dest_t, dest_f); + LLVMBuildCondBr(obj->builder, cgb->outflags, dest_t, dest_f); } else if (ir->cc == JIT_CC_F) { assert(cgb->source->out.count == 2); LLVMBasicBlockRef dest_t = - req->blocks[jit_get_edge(&(cgb->source->out), 1)].bbref; + cgb->func->blocks[jit_get_edge(&(cgb->source->out), 1)].bbref; LLVMBasicBlockRef dest_f = (cgb + 1)->bbref; - LLVMBuildCondBr(req->builder, cgb->outflags, dest_f, dest_t); - } - else { - jit_dump(req->func); - fatal_trace("unhandled jump condition code"); + LLVMBuildCondBr(obj->builder, cgb->outflags, dest_f, dest_t); } + else + cgen_abort(cgb, ir, "unhandled jump condition code"); } -static void cgen_op_cmp(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_cmp(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef arg1 = cgen_get_value(req, cgb, ir->arg1); - LLVMValueRef arg2 = cgen_get_value(req, cgb, ir->arg2); + LLVMValueRef arg1 = cgen_get_value(obj, cgb, ir->arg1); + LLVMValueRef arg2 = cgen_get_value(obj, cgb, ir->arg2); LLVMIntPredicate pred; switch (ir->cc) { @@ -1016,17 +1161,16 @@ static void cgen_op_cmp(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) case JIT_CC_LE: pred = LLVMIntSLE; break; case JIT_CC_GE: pred = LLVMIntSGE; break; default: - jit_dump_with_mark(req->func, ir - req->func->irbuf, false); - fatal_trace("unhandled cmp condition code"); + cgen_abort(cgb, ir, "unhandled cmp condition code"); } - cgb->outflags = LLVMBuildICmp(req->builder, pred, arg1, arg2, "FLAGS"); + cgb->outflags = LLVMBuildICmp(obj->builder, pred, arg1, arg2, "FLAGS"); } -static void cgen_op_fcmp(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_fcmp(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef arg1 = cgen_coerce_value(req, cgb, ir->arg1, LLVM_DOUBLE); - LLVMValueRef arg2 = cgen_coerce_value(req, cgb, ir->arg2, LLVM_DOUBLE); + LLVMValueRef arg1 = cgen_coerce_value(obj, cgb, ir->arg1, LLVM_DOUBLE); + LLVMValueRef arg2 = cgen_coerce_value(obj, cgb, ir->arg2, LLVM_DOUBLE); LLVMRealPredicate pred; switch (ir->cc) { @@ -1037,354 +1181,341 @@ static void cgen_op_fcmp(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) case JIT_CC_LE: pred = LLVMRealULE; break; case JIT_CC_GE: pred = LLVMRealUGE; break; default: - jit_dump_with_mark(req->func, ir - req->func->irbuf, false); - fatal_trace("unhandled fcmp condition code"); + cgen_abort(cgb, ir, "unhandled fcmp condition code"); } - cgb->outflags = LLVMBuildFCmp(req->builder, pred, arg1, arg2, "FLAGS"); + cgb->outflags = LLVMBuildFCmp(obj->builder, pred, arg1, arg2, "FLAGS"); } -static void cgen_op_cset(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_cset(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - cgen_zext_result(req, cgb, ir, cgb->outflags); + cgen_zext_result(obj, cgb, ir, cgb->outflags); } -static void cgen_op_csel(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_csel(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef arg1 = cgen_get_value(req, cgb, ir->arg1); - LLVMValueRef arg2 = cgen_get_value(req, cgb, ir->arg2); + LLVMValueRef arg1 = cgen_get_value(obj, cgb, ir->arg1); + LLVMValueRef arg2 = cgen_get_value(obj, cgb, ir->arg2); LLVMValueRef result = - LLVMBuildSelect(req->builder, cgb->outflags, arg1, arg2, ""); + LLVMBuildSelect(obj->builder, cgb->outflags, arg1, arg2, ""); - cgen_sext_result(req, cgb, ir, result); + cgen_sext_result(obj, cgb, ir, result); } -static void cgen_op_call(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_call(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - cgen_sync_irpos(req, ir); + cgen_sync_irpos(obj, cgb, ir); - jit_func_t *callee = jit_get_func(req->func->jit, ir->arg1.handle); - const char *name = cgen_istr(req, callee->name); - LLVMValueRef global = LLVMGetNamedGlobal(req->module, name); - if (global == NULL) { - global = LLVMAddGlobal(req->module, req->types[LLVM_PTR], name); - LLVMSetGlobalConstant(global, true); - LLVMSetLinkage(global, LLVMPrivateLinkage); - LLVMSetUnnamedAddr(global, true); - LLVMSetInitializer(global, llvm_ptr(req, callee->entry)); - } + jit_func_t *callee = jit_get_func(cgb->func->source->jit, ir->arg1.handle); + + LLVMValueRef fnptr = NULL; + if (obj->aot) + fnptr = llvm_get_fn(obj, LLVM_TRAMPOLINE); + else { + const char *name = cgen_istr(obj, callee->name); + LLVMValueRef global = LLVMGetNamedGlobal(obj->module, name); + if (global == NULL) { + global = LLVMAddGlobal(obj->module, obj->types[LLVM_PTR], name); + LLVMSetGlobalConstant(global, true); + LLVMSetLinkage(global, LLVMPrivateLinkage); + LLVMSetUnnamedAddr(global, true); + LLVMSetInitializer(global, llvm_ptr(obj, callee->entry)); + } #ifdef LLVM_HAS_OPAQUE_POINTERS - LLVMValueRef fnptr = - LLVMBuildLoad2(req->builder, req->types[LLVM_PTR], global, name); + fnptr = LLVMBuildLoad2(obj->builder, obj->types[LLVM_PTR], global, name); #else - LLVMTypeRef ptr_type = LLVMPointerType(req->types[LLVM_ENTRY_FN], 0); - LLVMValueRef cast = LLVMBuildPointerCast(req->builder, global, ptr_type, ""); - LLVMValueRef fnptr = LLVMBuildLoad2(req->builder, ptr_type, cast, name); + LLVMTypeRef ptr_type = LLVMPointerType(obj->types[LLVM_ENTRY_FN], 0); + LLVMValueRef cast = + LLVMBuildPointerCast(obj->builder, global, ptr_type, ""); + fnptr = LLVMBuildLoad2(obj->builder, ptr_type, cast, name); #endif + } LLVMValueRef args[] = { - llvm_ptr(req, callee), - PTR(req->anchor), - req->args + llvm_ptr(obj, callee), + PTR(cgb->func->anchor), + cgb->func->args }; - LLVMBuildCall2(req->builder, req->types[LLVM_ENTRY_FN], fnptr, + LLVMBuildCall2(obj->builder, obj->types[LLVM_ENTRY_FN], fnptr, args, ARRAY_LEN(args), ""); } -static void cgen_op_lea(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_lea(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef ptr = cgen_get_value(req, cgb, ir->arg1); + LLVMValueRef ptr = cgen_get_value(obj, cgb, ir->arg1); if (LLVMGetTypeKind(LLVMTypeOf(ptr)) == LLVMPointerTypeKind) - cgb->outregs[ir->result] = LLVMBuildPtrToInt(req->builder, ptr, - req->types[LLVM_INT64], + cgb->outregs[ir->result] = LLVMBuildPtrToInt(obj->builder, ptr, + obj->types[LLVM_INT64], cgen_reg_name(ir->result)); else - cgen_zext_result(req, cgb, ir, ptr); + cgen_zext_result(obj, cgb, ir, ptr); } -static void cgen_op_mov(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_mov(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef value = cgen_get_value(req, cgb, ir->arg1); - cgen_sext_result(req, cgb, ir, value); + LLVMValueRef value = cgen_get_value(obj, cgb, ir->arg1); + cgen_sext_result(obj, cgb, ir, value); } -static void cgen_op_neg(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_op_neg(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef arg1 = cgen_get_value(req, cgb, ir->arg1); + LLVMValueRef arg1 = cgen_get_value(obj, cgb, ir->arg1); LLVMValueRef neg = - LLVMBuildNeg(req->builder, arg1, cgen_reg_name(ir->result)); + LLVMBuildNeg(obj->builder, arg1, cgen_reg_name(ir->result)); cgb->outregs[ir->result] = neg; } -static void cgen_macro_exp(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_macro_exp(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef arg1 = cgen_get_value(req, cgb, ir->arg1); - LLVMValueRef arg2 = cgen_get_value(req, cgb, ir->arg2); + LLVMValueRef arg1 = cgen_get_value(obj, cgb, ir->arg1); + LLVMValueRef arg2 = cgen_get_value(obj, cgb, ir->arg2); // TODO: implement this without the cast - LLVMValueRef fn = cgen_get_fn(req, LLVM_POW_F64); LLVMValueRef cast[] = { - LLVMBuildUIToFP(req->builder, arg1, req->types[LLVM_DOUBLE], ""), - LLVMBuildUIToFP(req->builder, arg2, req->types[LLVM_DOUBLE], "") + LLVMBuildUIToFP(obj->builder, arg1, obj->types[LLVM_DOUBLE], ""), + LLVMBuildUIToFP(obj->builder, arg2, obj->types[LLVM_DOUBLE], "") }; - LLVMValueRef real = - LLVMBuildCall2(req->builder, req->fntypes[LLVM_POW_F64], fn, cast, 2, ""); + LLVMValueRef real = llvm_call_fn(obj, LLVM_POW_F64, cast, 2); cgb->outregs[ir->result] = LLVMBuildFPToUI( - req->builder, real, req->types[LLVM_INT64], cgen_reg_name(ir->result)); + obj->builder, real, obj->types[LLVM_INT64], cgen_reg_name(ir->result)); } -static void cgen_macro_fexp(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_macro_fexp(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef arg1 = cgen_coerce_value(req, cgb, ir->arg1, LLVM_DOUBLE); - LLVMValueRef arg2 = cgen_coerce_value(req, cgb, ir->arg2, LLVM_DOUBLE); + LLVMValueRef arg1 = cgen_coerce_value(obj, cgb, ir->arg1, LLVM_DOUBLE); + LLVMValueRef arg2 = cgen_coerce_value(obj, cgb, ir->arg2, LLVM_DOUBLE); - LLVMValueRef fn = cgen_get_fn(req, LLVM_POW_F64); LLVMValueRef args[] = { arg1, arg2 }; - LLVMValueRef real = LLVMBuildCall2(req->builder, req->fntypes[LLVM_POW_F64], - fn, args, ARRAY_LEN(args), ""); + LLVMValueRef real = llvm_call_fn(obj, LLVM_POW_F64, args, 2); - cgen_sext_result(req, cgb, ir, real); + cgen_sext_result(obj, cgb, ir, real); } -static void cgen_macro_copy(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_macro_copy(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { LLVMValueRef count = cgb->outregs[ir->result]; - LLVMValueRef dest = cgen_coerce_value(req, cgb, ir->arg1, LLVM_PTR); - LLVMValueRef src = cgen_coerce_value(req, cgb, ir->arg2, LLVM_PTR); + LLVMValueRef dest = cgen_coerce_value(obj, cgb, ir->arg1, LLVM_PTR); + LLVMValueRef src = cgen_coerce_value(obj, cgb, ir->arg2, LLVM_PTR); - LLVMBuildMemMove(req->builder, dest, 0, src, 0, count); + LLVMBuildMemMove(obj->builder, dest, 0, src, 0, count); } -static void cgen_macro_bzero(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_macro_bzero(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { LLVMValueRef count = cgb->outregs[ir->result]; - LLVMValueRef dest = cgen_coerce_value(req, cgb, ir->arg1, LLVM_PTR); + LLVMValueRef dest = cgen_coerce_value(obj, cgb, ir->arg1, LLVM_PTR); - LLVMBuildMemSet(req->builder, PTR(dest), llvm_int8(req, 0), count, 0); + LLVMBuildMemSet(obj->builder, PTR(dest), llvm_int8(obj, 0), count, 0); } -static void cgen_macro_exit(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_macro_exit(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - cgen_sync_irpos(req, ir); + cgen_sync_irpos(obj, cgb, ir); - LLVMValueRef which = cgen_get_value(req, cgb, ir->arg1); - LLVMValueRef fn = cgen_get_fn(req, LLVM_DO_EXIT); + LLVMValueRef which = cgen_get_value(obj, cgb, ir->arg1); LLVMValueRef args[] = { which, - PTR(req->anchor), - req->args + PTR(cgb->func->anchor), + cgb->func->args }; - LLVMBuildCall2(req->builder, req->fntypes[LLVM_DO_EXIT], fn, - args, ARRAY_LEN(args), ""); + llvm_call_fn(obj, LLVM_DO_EXIT, args, ARRAY_LEN(args)); } -static void cgen_macro_fficall(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_macro_fficall(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - const unsigned irpos = ir - req->func->irbuf; - LLVMValueRef irpos_ptr = LLVMBuildStructGEP2(req->builder, - req->types[LLVM_ANCHOR], - req->anchor, 2, ""); - LLVMBuildStore(req->builder, llvm_int32(req, irpos), irpos_ptr); + cgen_sync_irpos(obj, cgb, ir); - LLVMValueRef ff = cgen_get_value(req, cgb, ir->arg1); - LLVMValueRef fn = cgen_get_fn(req, LLVM_DO_FFICALL); + LLVMValueRef ff = cgen_get_value(obj, cgb, ir->arg1); LLVMValueRef args[] = { ff, - PTR(req->anchor), - req->args + PTR(cgb->func->anchor), + cgb->func->args }; - LLVMBuildCall2(req->builder, req->fntypes[LLVM_DO_FFICALL], fn, - args, ARRAY_LEN(args), ""); + llvm_call_fn(obj, LLVM_DO_FFICALL, args, ARRAY_LEN(args)); } -static void cgen_macro_galloc(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_macro_galloc(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { // TODO: use TLAB - LLVMValueRef fn = cgen_get_fn(req, LLVM_MSPACE_ALLOC); - LLVMValueRef size = cgen_get_value(req, cgb, ir->arg1); + cgen_sync_irpos(obj, cgb, ir); + + LLVMValueRef size = cgen_get_value(obj, cgb, ir->arg1); LLVMValueRef args[] = { - LLVMBuildTrunc(req->builder, size, req->types[LLVM_INT32], ""), - llvm_int32(req, 1), + LLVMBuildTrunc(obj->builder, size, obj->types[LLVM_INT32], ""), + llvm_int32(obj, 1), }; - LLVMValueRef ptr = LLVMBuildCall2(req->builder, - req->fntypes[LLVM_MSPACE_ALLOC], - fn, args, ARRAY_LEN(args), ""); + LLVMValueRef ptr = llvm_call_fn(obj, LLVM_MSPACE_ALLOC, args, + ARRAY_LEN(args)); - cgb->outregs[ir->result] = LLVMBuildPtrToInt(req->builder, ptr, - req->types[LLVM_INT64], + cgb->outregs[ir->result] = LLVMBuildPtrToInt(obj->builder, ptr, + obj->types[LLVM_INT64], cgen_reg_name(ir->result)); } -static void cgen_macro_getpriv(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_macro_getpriv(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { // TODO: this needs some kind of fast-path - LLVMValueRef fn = cgen_get_fn(req, LLVM_GETPRIV); - LLVMValueRef args[] = { - cgen_get_value(req, cgb, ir->arg1) + cgen_get_value(obj, cgb, ir->arg1) }; - LLVMValueRef ptr = LLVMBuildCall2(req->builder,req->fntypes[LLVM_GETPRIV], - fn, args, ARRAY_LEN(args), ""); + LLVMValueRef ptr = llvm_call_fn(obj, LLVM_GETPRIV, args, ARRAY_LEN(args)); - cgb->outregs[ir->result] = LLVMBuildPtrToInt(req->builder, ptr, - req->types[LLVM_INT64], + cgb->outregs[ir->result] = LLVMBuildPtrToInt(obj->builder, ptr, + obj->types[LLVM_INT64], cgen_reg_name(ir->result)); } -static void cgen_macro_putpriv(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_macro_putpriv(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef fn = cgen_get_fn(req, LLVM_PUTPRIV); - LLVMValueRef args[] = { - cgen_get_value(req, cgb, ir->arg1), - cgen_coerce_value(req, cgb, ir->arg2, LLVM_PTR), + cgen_get_value(obj, cgb, ir->arg1), + cgen_coerce_value(obj, cgb, ir->arg2, LLVM_PTR), }; - LLVMBuildCall2(req->builder, req->fntypes[LLVM_PUTPRIV], - fn, args, ARRAY_LEN(args), ""); + llvm_call_fn(obj, LLVM_PUTPRIV, args, ARRAY_LEN(args)); } -static void cgen_ir(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +static void cgen_ir(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { switch (ir->op) { case J_RECV: - cgen_op_recv(req, cgb, ir); + cgen_op_recv(obj, cgb, ir); break; case J_SEND: - cgen_op_send(req, cgb, ir); + cgen_op_send(obj, cgb, ir); break; case J_STORE: - cgen_op_store(req, cgb, ir); + cgen_op_store(obj, cgb, ir); break; case J_LOAD: case J_ULOAD: - cgen_op_load(req, cgb, ir); + cgen_op_load(obj, cgb, ir); break; case J_ADD: - cgen_op_add(req, cgb, ir); + cgen_op_add(obj, cgb, ir); break; case J_SUB: - cgen_op_sub(req, cgb, ir); + cgen_op_sub(obj, cgb, ir); break; case J_MUL: - cgen_op_mul(req, cgb, ir); + cgen_op_mul(obj, cgb, ir); break; case J_DIV: - cgen_op_div(req, cgb, ir); + cgen_op_div(obj, cgb, ir); break; case J_REM: - cgen_op_rem(req, cgb, ir); + cgen_op_rem(obj, cgb, ir); break; case J_FADD: - cgen_op_fadd(req, cgb, ir); + cgen_op_fadd(obj, cgb, ir); break; case J_FSUB: - cgen_op_fsub(req, cgb, ir); + cgen_op_fsub(obj, cgb, ir); break; case J_FMUL: - cgen_op_fmul(req, cgb, ir); + cgen_op_fmul(obj, cgb, ir); break; case J_FDIV: - cgen_op_fdiv(req, cgb, ir); + cgen_op_fdiv(obj, cgb, ir); break; case J_FNEG: - cgen_op_fneg(req, cgb, ir); + cgen_op_fneg(obj, cgb, ir); break; case J_FCVTNS: - cgen_op_fcvtns(req, cgb, ir); + cgen_op_fcvtns(obj, cgb, ir); break; case J_SCVTF: - cgen_op_scvtf(req, cgb, ir); + cgen_op_scvtf(obj, cgb, ir); break; case J_NOT: - cgen_op_not(req, cgb, ir); + cgen_op_not(obj, cgb, ir); break; case J_AND: - cgen_op_and(req, cgb, ir); + cgen_op_and(obj, cgb, ir); break; case J_OR: - cgen_op_or(req, cgb, ir); + cgen_op_or(obj, cgb, ir); break; case J_XOR: - cgen_op_xor(req, cgb, ir); + cgen_op_xor(obj, cgb, ir); break; case J_RET: - cgen_op_ret(req, ir); + cgen_op_ret(obj, ir); break; case J_JUMP: - cgen_op_jump(req, cgb, ir); + cgen_op_jump(obj, cgb, ir); break; case J_CMP: - cgen_op_cmp(req, cgb, ir); + cgen_op_cmp(obj, cgb, ir); break; case J_FCMP: - cgen_op_fcmp(req, cgb, ir); + cgen_op_fcmp(obj, cgb, ir); break; case J_CSET: - cgen_op_cset(req, cgb, ir); + cgen_op_cset(obj, cgb, ir); break; case J_CSEL: - cgen_op_csel(req, cgb, ir); + cgen_op_csel(obj, cgb, ir); break; case J_DEBUG: break; case J_CALL: - cgen_op_call(req, cgb, ir); + cgen_op_call(obj, cgb, ir); break; case J_LEA: - cgen_op_lea(req, cgb, ir); + cgen_op_lea(obj, cgb, ir); break; case J_MOV: - cgen_op_mov(req, cgb, ir); + cgen_op_mov(obj, cgb, ir); break; case J_NEG: - cgen_op_neg(req, cgb, ir); + cgen_op_neg(obj, cgb, ir); break; case MACRO_EXP: - cgen_macro_exp(req, cgb, ir); + cgen_macro_exp(obj, cgb, ir); break; case MACRO_FEXP: - cgen_macro_fexp(req, cgb, ir); + cgen_macro_fexp(obj, cgb, ir); break; case MACRO_COPY: - cgen_macro_copy(req, cgb, ir); + cgen_macro_copy(obj, cgb, ir); break; case MACRO_BZERO: - cgen_macro_bzero(req, cgb, ir); + cgen_macro_bzero(obj, cgb, ir); break; case MACRO_EXIT: - cgen_macro_exit(req, cgb, ir); + cgen_macro_exit(obj, cgb, ir); break; case MACRO_FFICALL: - cgen_macro_fficall(req, cgb, ir); + cgen_macro_fficall(obj, cgb, ir); break; case MACRO_GALLOC: - cgen_macro_galloc(req, cgb, ir); + cgen_macro_galloc(obj, cgb, ir); break; case MACRO_GETPRIV: - cgen_macro_getpriv(req, cgb, ir); + cgen_macro_getpriv(obj, cgb, ir); break; case MACRO_PUTPRIV: - cgen_macro_putpriv(req, cgb, ir); + cgen_macro_putpriv(obj, cgb, ir); break; default: - jit_dump_with_mark(req->func, ir - req->func->irbuf, false); - fatal("cannot generate LLVM for %s", jit_op_name(ir->op)); + cgen_abort(cgb, ir, "cannot generate LLVM for %s", jit_op_name(ir->op)); } } -static void cgen_basic_blocks(cgen_req_t *req, jit_cfg_t *cfg) +static void cgen_basic_blocks(llvm_obj_t *obj, cgen_func_t *func, + jit_cfg_t *cfg) { - req->blocks = xcalloc_array(cfg->nblocks, sizeof(cgen_block_t)); + func->blocks = xcalloc_array(cfg->nblocks, sizeof(cgen_block_t)); for (int i = 0; i < cfg->nblocks; i++) { #ifdef DEBUG @@ -1394,139 +1525,80 @@ static void cgen_basic_blocks(cgen_req_t *req, jit_cfg_t *cfg) const char *name = ""; #endif - cgen_block_t *cgb = &(req->blocks[i]); - cgb->bbref = cgen_append_block(req, name); + cgen_block_t *cgb = &(func->blocks[i]); + cgb->bbref = llvm_append_block(obj, func->llvmfn, name); cgb->source = &(cfg->blocks[i]); + cgb->func = func; - cgb->inregs = xcalloc_array(req->func->nregs, sizeof(LLVMValueRef)); - cgb->outregs = xcalloc_array(req->func->nregs, sizeof(LLVMValueRef)); + cgb->inregs = xcalloc_array(func->source->nregs, sizeof(LLVMValueRef)); + cgb->outregs = xcalloc_array(func->source->nregs, sizeof(LLVMValueRef)); } } -static void cgen_dump_module(cgen_req_t *req, const char *tag) +static void cgen_frame_anchor(llvm_obj_t *obj, cgen_func_t *func) { - size_t length; - const char *module_name = LLVMGetModuleIdentifier(req->module, &length); - - if (!opt_get_verbose(OPT_LLVM_VERBOSE, module_name)) - return; + LLVMTypeRef type = obj->types[LLVM_ANCHOR]; + func->anchor = LLVMBuildAlloca(obj->builder, type, "anchor"); - LOCAL_TEXT_BUF tb = tb_new(); - tb_printf(tb, "%s.%s.ll", module_name, tag); - - char *error; - if (LLVMPrintModuleToFile(req->module, tb_get(tb), &error)) - fatal("Failed to write LLVM IR file: %s", error); + LLVMValueRef func_arg = LLVMGetParam(func->llvmfn, 0); + LLVMSetValueName(func_arg, "func"); - debugf("wrote LLVM IR for %s to %s", module_name, tb_get(tb)); -} + LLVMValueRef caller_arg = LLVMGetParam(func->llvmfn, 1); + LLVMSetValueName(caller_arg, "caller"); -static void cgen_frame_anchor(cgen_req_t *req) -{ - LLVMTypeRef type = req->types[LLVM_ANCHOR]; - req->anchor = LLVMBuildAlloca(req->builder, type, "anchor"); + LLVMValueRef caller_ptr = LLVMBuildStructGEP2(obj->builder, type, + func->anchor, 0, ""); + LLVMBuildStore(obj->builder, caller_arg, caller_ptr); - LLVMValueRef func = LLVMGetParam(req->llvmfn, 0); - LLVMSetValueName(func, "func"); + LLVMValueRef func_ptr = LLVMBuildStructGEP2(obj->builder, type, + func->anchor, 1, ""); + LLVMBuildStore(obj->builder, func_arg, func_ptr); - LLVMValueRef caller = LLVMGetParam(req->llvmfn, 1); - LLVMSetValueName(caller, "caller"); - - LLVMValueRef caller_ptr = LLVMBuildStructGEP2(req->builder, type, - req->anchor, 0, ""); - LLVMBuildStore(req->builder, caller, caller_ptr); - - LLVMValueRef func_ptr = LLVMBuildStructGEP2(req->builder, type, - req->anchor, 1, ""); - LLVMBuildStore(req->builder, func, func_ptr); - - LLVMValueRef irpos_ptr = LLVMBuildStructGEP2(req->builder, type, - req->anchor, 2, ""); - LLVMBuildStore(req->builder, llvm_int32(req, 0), irpos_ptr); + LLVMValueRef irpos_ptr = LLVMBuildStructGEP2(obj->builder, type, + func->anchor, 2, ""); + LLVMBuildStore(obj->builder, llvm_int32(obj, 0), irpos_ptr); } -static void cgen_module(cgen_req_t *req) +static void cgen_module(llvm_obj_t *obj, cgen_func_t *func) { - req->module = LLVMModuleCreateWithNameInContext(req->name, req->context); - req->builder = LLVMCreateBuilderInContext(req->context); - - char *triple = LLVMGetTargetMachineTriple(req->target); - LLVMSetTarget(req->module, triple); - LLVMDisposeMessage(triple); - - LLVMTargetDataRef data_ref = LLVMCreateTargetDataLayout(req->target); - LLVMSetModuleDataLayout(req->module, data_ref); - - req->types[LLVM_VOID] = LLVMVoidTypeInContext(req->context); - req->types[LLVM_INT1] = LLVMInt1TypeInContext(req->context); - req->types[LLVM_INT8] = LLVMInt8TypeInContext(req->context); - req->types[LLVM_INT16] = LLVMInt16TypeInContext(req->context); - req->types[LLVM_INT32] = LLVMInt32TypeInContext(req->context); - req->types[LLVM_INT64] = LLVMInt64TypeInContext(req->context); - req->types[LLVM_INTPTR] = LLVMIntPtrTypeInContext(req->context, data_ref); - req->types[LLVM_DOUBLE] = LLVMDoubleTypeInContext(req->context); - - -#ifdef LLVM_HAS_OPAQUE_POINTERS - req->types[LLVM_PTR] = LLVMPointerTypeInContext(req->context, 0); -#else - req->types[LLVM_PTR] = LLVMPointerType(req->types[LLVM_INT8], 0); -#endif - - LLVMTypeRef atypes[] = { - req->types[LLVM_PTR], // Function - req->types[LLVM_PTR], // Anchor - req->types[LLVM_PTR] // Arguments - }; - req->types[LLVM_ENTRY_FN] = LLVMFunctionType(req->types[LLVM_VOID], atypes, - ARRAY_LEN(atypes), false); - - LLVMTypeRef fields[] = { - req->types[LLVM_PTR], // Caller - req->types[LLVM_PTR], // Function - req->types[LLVM_INT32] // IR position - }; - req->types[LLVM_ANCHOR] = LLVMStructTypeInContext(req->context, fields, - ARRAY_LEN(fields), false); - - req->llvmfn = LLVMAddFunction(req->module, req->name, - req->types[LLVM_ENTRY_FN]); + func->llvmfn = LLVMAddFunction(obj->module, func->name, + obj->types[LLVM_ENTRY_FN]); - LLVMBasicBlockRef entry_bb = cgen_append_block(req, "entry"); - LLVMPositionBuilderAtEnd(req->builder, entry_bb); + LLVMBasicBlockRef entry_bb = llvm_append_block(obj, func->llvmfn, "entry"); + LLVMPositionBuilderAtEnd(obj->builder, entry_bb); - cgen_frame_anchor(req); + cgen_frame_anchor(obj, func); - req->args = LLVMGetParam(req->llvmfn, 2); - LLVMSetValueName(req->args, "args"); + func->args = LLVMGetParam(func->llvmfn, 2); + LLVMSetValueName(func->args, "args"); - if (req->func->framesz > 0) { + if (func->source->framesz > 0) { LLVMTypeRef frame_type = - LLVMArrayType(req->types[LLVM_INT8], req->func->framesz); - req->frame = LLVMBuildAlloca(req->builder, frame_type, "frame"); - LLVMSetAlignment(req->frame, sizeof(double)); + LLVMArrayType(obj->types[LLVM_INT8], func->source->framesz); + func->frame = LLVMBuildAlloca(obj->builder, frame_type, "frame"); + LLVMSetAlignment(func->frame, sizeof(double)); } - jit_cfg_t *cfg = req->cfg = jit_get_cfg(req->func); - cgen_basic_blocks(req, cfg); + jit_cfg_t *cfg = func->cfg = jit_get_cfg(func->source); + cgen_basic_blocks(obj, func, cfg); - cgen_block_t *cgb = req->blocks; + cgen_block_t *cgb = func->blocks; int maxin = 0; - for (int i = 0; i < req->func->nirs; i++) { + for (int i = 0; i < func->source->nirs; i++) { if (i == cgb->source->first) { - LLVMPositionBuilderAtEnd(req->builder, cgb->bbref); + LLVMPositionBuilderAtEnd(obj->builder, cgb->bbref); - LLVMTypeRef int1_type = req->types[LLVM_INT1]; - cgb->inflags = LLVMBuildPhi(req->builder, int1_type, "FLAGS"); + LLVMTypeRef int1_type = obj->types[LLVM_INT1]; + cgb->inflags = LLVMBuildPhi(obj->builder, int1_type, "FLAGS"); cgb->outflags = cgb->inflags; - for (int j = 0; j < req->func->nregs; j++) { + for (int j = 0; j < func->source->nregs; j++) { if (mask_test(&cgb->source->livein, j)) { const char *name = cgen_reg_name(j); LLVMValueRef init = i == 0 // Entry block - ? LLVMConstNull(req->types[LLVM_INT64]) - : LLVMBuildPhi(req->builder, req->types[LLVM_INT64], name); + ? LLVMConstNull(obj->types[LLVM_INT64]) + : LLVMBuildPhi(obj->builder, obj->types[LLVM_INT64], name); cgb->inregs[j] = cgb->outregs[j] = init; } } @@ -1536,26 +1608,26 @@ static void cgen_module(cgen_req_t *req) assert(i >= cgb->source->first && i <= cgb->source->last); - cgen_ir(req, cgb, &(req->func->irbuf[i])); + cgen_ir(obj, cgb, &(func->source->irbuf[i])); if (i == cgb->source->last) { if (cgb->source->aborts) - LLVMBuildUnreachable(req->builder); + LLVMBuildUnreachable(obj->builder); if (LLVMGetBasicBlockTerminator(cgb->bbref) == NULL) { // Fall through to next block assert(!cgb->source->returns); - assert(cgb + 1 < req->blocks + cfg->nblocks); - LLVMBuildBr(req->builder, (++cgb)->bbref); + assert(cgb + 1 < func->blocks + cfg->nblocks); + LLVMBuildBr(obj->builder, (++cgb)->bbref); } else ++cgb; } } - LLVMValueRef flags0_in[] = { llvm_int1(req, false) }; + LLVMValueRef flags0_in[] = { llvm_int1(obj, false) }; LLVMBasicBlockRef flags0_bb[] = { entry_bb }; - LLVMAddIncoming(req->blocks[0].inflags, flags0_in, flags0_bb, 1); + LLVMAddIncoming(func->blocks[0].inflags, flags0_in, flags0_bb, 1); LLVMValueRef *phi_in LOCAL = xmalloc_array(maxin, sizeof(LLVMValueRef)); LLVMBasicBlockRef *phi_bb LOCAL = @@ -1563,24 +1635,24 @@ static void cgen_module(cgen_req_t *req) for (int i = 0; i < cfg->nblocks; i++) { jit_block_t *bb = &(cfg->blocks[i]); - cgen_block_t *cgb = &(req->blocks[i]); + cgen_block_t *cgb = &(func->blocks[i]); // Flags for (int j = 0; j < bb->in.count; j++) { const int edge = jit_get_edge(&bb->in, j); - phi_in[j] = req->blocks[edge].outflags; - phi_bb[j] = req->blocks[edge].bbref; + phi_in[j] = func->blocks[edge].outflags; + phi_bb[j] = func->blocks[edge].bbref; } LLVMAddIncoming(cgb->inflags, phi_in, phi_bb, bb->in.count); // Live-in registers - for (int j = 0; j < req->func->nregs; j++) { + for (int j = 0; j < func->source->nregs; j++) { if (cgb->inregs[j] != NULL) { for (int k = 0; k < bb->in.count; k++) { const int edge = jit_get_edge(&bb->in, k); - assert(req->blocks[edge].outregs[j] != NULL); - phi_in[k] = req->blocks[edge].outregs[j]; - phi_bb[k] = req->blocks[edge].bbref; + assert(func->blocks[edge].outregs[j] != NULL); + phi_in[k] = func->blocks[edge].outregs[j]; + phi_bb[k] = func->blocks[edge].bbref; } LLVMAddIncoming(cgb->inregs[j], phi_in, phi_bb, bb->in.count); } @@ -1588,83 +1660,41 @@ static void cgen_module(cgen_req_t *req) } for (int i = 0; i < cfg->nblocks; i++) { - cgen_block_t *cgb = &(req->blocks[i]); + cgen_block_t *cgb = &(func->blocks[i]); free(cgb->inregs); free(cgb->outregs); cgb->inregs = cgb->outregs = NULL; } - LLVMPositionBuilderAtEnd(req->builder, entry_bb); - LLVMBuildBr(req->builder, req->blocks[0].bbref); - - jit_free_cfg(req->func); - req->cfg = cfg = NULL; - - free(req->blocks); - req->blocks = NULL; - - LLVMDisposeBuilder(req->builder); - req->builder = NULL; - - LLVMDisposeTargetData(data_ref); + LLVMPositionBuilderAtEnd(obj->builder, entry_bb); + LLVMBuildBr(obj->builder, func->blocks[0].bbref); - cgen_dump_module(req, "initial"); + jit_free_cfg(func->source); + func->cfg = cfg = NULL; -#ifdef DEBUG - if (LLVMVerifyModule(req->module, LLVMPrintMessageAction, NULL)) - fatal("LLVM verification failed for %s", cgen_istr(req, req->func->name)); -#endif + free(func->blocks); + func->blocks = NULL; } -static void cgen_optimise(cgen_req_t *req) -{ - LLVMPassManagerRef fpm = LLVMCreateFunctionPassManagerForModule(req->module); +//////////////////////////////////////////////////////////////////////////////// +// JIT plugin interface - LLVMAddScalarReplAggregatesPass(fpm); - LLVMAddInstructionCombiningPass(fpm); - LLVMAddReassociatePass(fpm); - LLVMAddGVNPass(fpm); - LLVMAddCFGSimplificationPass(fpm); - - LLVMInitializeFunctionPassManager(fpm); - - for (LLVMValueRef fn = LLVMGetFirstFunction(req->module); - fn != NULL; fn = LLVMGetNextFunction(fn)) - LLVMRunFunctionPassManager(fpm, fn); - - LLVMFinalizeFunctionPassManager(fpm); - LLVMDisposePassManager(fpm); - - cgen_dump_module(req, "final"); -} - -static LLVMTargetMachineRef llvm_target_machine(void) -{ - static __thread LLVMTargetMachineRef tm_ref = NULL; - if (tm_ref == NULL) { - char *def_triple = LLVMGetDefaultTargetTriple(); - char *error; - LLVMTargetRef target_ref; - if (LLVMGetTargetFromTriple(def_triple, &target_ref, &error)) - fatal("failed to get LLVM target for %s: %s", def_triple, error); - - tm_ref = LLVMCreateTargetMachine(target_ref, def_triple, "", "", - LLVMCodeGenLevelDefault, - LLVMRelocDefault, - LLVMCodeModelJITDefault); - LLVMDisposeMessage(def_triple); - } +#ifdef LLVM_HAS_LLJIT - return tm_ref; -} +typedef struct { + LLVMOrcThreadSafeContextRef context; + LLVMOrcLLJITRef jit; + LLVMOrcExecutionSessionRef session; + LLVMOrcJITDylibRef dylib; +} lljit_state_t; static void *jit_llvm_init(void) { - lljit_state_t *state = xcalloc(sizeof(lljit_state_t)); - LLVMInitializeNativeTarget(); LLVMInitializeNativeAsmPrinter(); + lljit_state_t *state = xcalloc(sizeof(lljit_state_t)); + LLVMOrcLLJITBuilderRef builder = LLVMOrcCreateLLJITBuilder(); LLVM_CHECK(LLVMOrcCreateLLJIT, &state->jit, builder); @@ -1696,36 +1726,49 @@ static void jit_llvm_cgen(jit_t *j, jit_handle_t handle, void *context) const uint64_t start_us = get_timestamp_us(); - LOCAL_TEXT_BUF tb = tb_new(); - tb_istr(tb, f->name); - - cgen_req_t req = { + llvm_obj_t obj = { .context = LLVMOrcThreadSafeContextGetContext(state->context), .target = llvm_target_machine(), - .name = tb_claim(tb), - .func = f, .textbuf = tb_new(), }; - cgen_module(&req); - cgen_optimise(&req); + + LOCAL_TEXT_BUF tb = tb_new(); + tb_istr(tb, f->name); + + obj.module = LLVMModuleCreateWithNameInContext(tb_get(tb), obj.context); + obj.builder = LLVMCreateBuilderInContext(obj.context); + obj.data_ref = LLVMCreateTargetDataLayout(obj.target); + + llvm_register_types(&obj); + + cgen_func_t func = { + .name = tb_claim(tb), + .source = f, + }; + + cgen_module(&obj, &func); + + llvm_finalise(&obj); LLVMOrcThreadSafeModuleRef tsm = - LLVMOrcCreateNewThreadSafeModule(req.module, state->context); + LLVMOrcCreateNewThreadSafeModule(obj.module, state->context); LLVMOrcLLJITAddLLVMIRModule(state->jit, state->dylib, tsm); LLVMOrcJITTargetAddress addr; - LLVM_CHECK(LLVMOrcLLJITLookup, state->jit, &addr, req.name); + LLVM_CHECK(LLVMOrcLLJITLookup, state->jit, &addr, func.name); const uint64_t end_us = get_timestamp_us(); static __thread uint64_t slowest = 0; if (end_us - start_us > slowest) - debugf("%s at %p [%"PRIi64" us]", req.name, (void *)addr, + debugf("%s at %p [%"PRIi64" us]", func.name, (void *)addr, (slowest = end_us - start_us)); - atomic_store(&f->entry, (jit_entry_fn_t)addr); + store_release(&f->entry, (jit_entry_fn_t)addr); - tb_free(req.textbuf); - free(req.name); + LLVMDisposeTargetData(obj.data_ref); + LLVMDisposeBuilder(obj.builder); + tb_free(obj.textbuf); + free(func.name); } static void jit_llvm_cleanup(void *context) @@ -1755,13 +1798,37 @@ void jit_register_llvm_plugin(jit_t *j) warnf("invalid NVC_JIT_THRESOLD setting %d", threshold); } -static inline LLVMContextRef llvm_context(void) +#endif // LLVM_HAS_LLJIT + +//////////////////////////////////////////////////////////////////////////////// +// Ahead-of-time code generation + +llvm_obj_t *llvm_obj_new(const char *name) { - static __thread LLVMContextRef context = NULL; - return context ?: (context = LLVMContextCreate()); + LLVMInitializeNativeTarget(); + LLVMInitializeNativeAsmPrinter(); + + llvm_obj_t *obj = xcalloc(sizeof(llvm_obj_t)); + obj->context = LLVMContextCreate(); + obj->module = LLVMModuleCreateWithNameInContext(name, obj->context); + obj->builder = LLVMCreateBuilderInContext(obj->context); + obj->target = llvm_target_machine(); + obj->textbuf = tb_new(); + obj->data_ref = LLVMCreateTargetDataLayout(obj->target); + obj->aot = true; + + char *triple = LLVMGetTargetMachineTriple(obj->target); + LLVMSetTarget(obj->module, triple); + LLVMDisposeMessage(triple); + + LLVMSetModuleDataLayout(obj->module, obj->data_ref); + + llvm_register_types(obj); + + return obj; } -LLVMModuleRef jit_llvm_for_aot(jit_t *j, jit_handle_t handle) +void llvm_aot_compile(llvm_obj_t *obj, jit_t *j, jit_handle_t handle) { jit_func_t *f = jit_get_func(j, handle); if (f->irbuf == NULL) @@ -1772,27 +1839,44 @@ LLVMModuleRef jit_llvm_for_aot(jit_t *j, jit_handle_t handle) LOCAL_TEXT_BUF tb = tb_new(); tb_istr(tb, f->name); - cgen_req_t req = { - .context = llvm_context(), - .target = llvm_target_machine(), - .name = tb_claim(tb), - .textbuf = tb_new(), - .func = f, + cgen_func_t func = { + .name = tb_claim(tb), + .source = f, }; - cgen_module(&req); - cgen_optimise(&req); + cgen_module(obj, &func); const uint64_t end_us = get_timestamp_us(); static __thread uint64_t slowest = 0; if (end_us - start_us > slowest) - debugf("compiled %s [%"PRIi64" us]", req.name, + debugf("compiled %s [%"PRIi64" us]", func.name, (slowest = end_us - start_us)); - tb_free(req.textbuf); - free(req.name); - - return req.module; + free(func.name); } -#endif // LLVM_HAS_LLJIT +void llvm_obj_emit(llvm_obj_t *obj, const char *file) +{ + llvm_finalise(obj); + + // TODO: emit to file + + LLVMDumpModule(obj->module); + + LLVMDisposeTargetData(obj->data_ref); + obj->data_ref = NULL; + + LLVMDisposeBuilder(obj->builder); + obj->builder = NULL; + + LLVMDisposeModule(obj->module); + obj->module = NULL; + + LLVMContextDispose(obj->context); + obj->context = NULL; + + tb_free(obj->textbuf); + obj->textbuf = NULL; + + free(obj); +} diff --git a/src/jit/jit-llvm.h b/src/jit/jit-llvm.h index 96cd2390..78d7201e 100644 --- a/src/jit/jit-llvm.h +++ b/src/jit/jit-llvm.h @@ -25,8 +25,10 @@ void jit_register_llvm_plugin(jit_t *j); #endif -typedef struct LLVMOpaqueModule *LLVMModuleRef; +typedef struct _llvm_obj llvm_obj_t; -LLVMModuleRef jit_llvm_for_aot(jit_t *j, jit_handle_t handle); +llvm_obj_t *llvm_obj_new(const char *name); +void llvm_aot_compile(llvm_obj_t *obj, jit_t *j, jit_handle_t handle); +void llvm_obj_emit(llvm_obj_t *obj, const char *file); #endif // _JIT_LLVM_H diff --git a/src/jit/jit-priv.h b/src/jit/jit-priv.h index 750f71b1..b45d9a5d 100644 --- a/src/jit/jit-priv.h +++ b/src/jit/jit-priv.h @@ -163,6 +163,7 @@ typedef enum { JIT_VALUE_HANDLE, JIT_VALUE_EXIT, JIT_VALUE_LOC, + JIT_VALUE_FOREIGN, } jit_value_kind_t; typedef uint32_t jit_label_t; @@ -172,13 +173,14 @@ typedef struct { jit_value_kind_t kind : 8; int32_t disp; union { - jit_reg_t reg; - int64_t int64; - double dval; - jit_label_t label; - jit_handle_t handle; - jit_exit_t exit; - loc_t loc; + jit_reg_t reg; + int64_t int64; + double dval; + jit_label_t label; + jit_handle_t handle; + jit_exit_t exit; + loc_t loc; + jit_foreign_t *foreign; }; } jit_value_t; -- 2.39.2