From 7f5f1f0408de406f52a72eda2a45b6f321522af9 Mon Sep 17 00:00:00 2001 From: Nick Gasson Date: Thu, 27 Oct 2022 11:37:45 +0100 Subject: [PATCH] Add FFI support to LLVM JIT --- src/jit/jit-exits.c | 12 +++++ src/jit/jit-interp.c | 3 +- src/jit/jit-llvm.c | 111 +++++++++++++++++++++++++++++++++---------- src/jit/jit-priv.h | 2 + src/symbols.txt | 1 + 5 files changed, 104 insertions(+), 25 deletions(-) diff --git a/src/jit/jit-exits.c b/src/jit/jit-exits.c index 1abb4fed..3725f031 100644 --- a/src/jit/jit-exits.c +++ b/src/jit/jit-exits.c @@ -1058,6 +1058,18 @@ void __nvc_do_exit(jit_exit_t which, jit_anchor_t *anchor, jit_scalar_t *args) thread->anchor = NULL; } +DLLEXPORT +void __nvc_do_fficall(jit_foreign_t *ff, jit_anchor_t *anchor, + jit_scalar_t *args) +{ + jit_thread_local_t *thread = jit_thread_local(); + thread->anchor = anchor; + + jit_ffi_call(ff, args); + + thread->anchor = NULL; +} + //////////////////////////////////////////////////////////////////////////////// // Entry points from AOT compiled code diff --git a/src/jit/jit-interp.c b/src/jit/jit-interp.c index 48f30710..9db82d8f 100644 --- a/src/jit/jit-interp.c +++ b/src/jit/jit-interp.c @@ -714,7 +714,8 @@ static void interp_exit(jit_interp_t *state, jit_ir_t *ir) static void interp_fficall(jit_interp_t *state, jit_ir_t *ir) { jit_foreign_t *ff = interp_get_value(state, ir->arg1).pointer; - jit_ffi_call(ff, state->args); + state->anchor->irpos = ir - state->func->irbuf; + __nvc_do_fficall(ff, state->anchor, state->args); } static void interp_getpriv(jit_interp_t *state, jit_ir_t *ir) diff --git a/src/jit/jit-llvm.c b/src/jit/jit-llvm.c index 339bdd7d..76613b01 100644 --- a/src/jit/jit-llvm.c +++ b/src/jit/jit-llvm.c @@ -93,6 +93,7 @@ typedef enum { LLVM_GETPRIV, LLVM_PUTPRIV, LLVM_MSPACE_ALLOC, + LLVM_DO_FFICALL, LLVM_LAST_FN, } llvm_fn_t; @@ -350,6 +351,21 @@ static LLVMValueRef cgen_get_fn(cgen_req_t *req, llvm_fn_t which) } break; + case LLVM_DO_FFICALL: + { + LLVMTypeRef args[] = { + req->types[LLVM_PTR], + req->types[LLVM_PTR], + req->types[LLVM_PTR] + }; + req->fntypes[which] = LLVMFunctionType(req->types[LLVM_VOID], args, + ARRAY_LEN(args), false); + + fn = LLVMAddFunction(req->module, "__nvc_do_fficall", + req->fntypes[which]); + } + break; + case LLVM_GETPRIV: { LLVMTypeRef args[] = { req->types[LLVM_INT32] }; @@ -441,6 +457,7 @@ static LLVMValueRef cgen_get_value(cgen_req_t *req, cgen_block_t *cgb, switch (value.kind) { case JIT_VALUE_REG: assert(value.reg < req->func->nregs); + assert(cgb->outregs[value.reg] != NULL); return cgb->outregs[value.reg]; case JIT_VALUE_INT64: return llvm_int64(req, value.int64); @@ -460,9 +477,22 @@ static LLVMValueRef cgen_get_value(cgen_req_t *req, cgen_block_t *cgb, assert(value.int64 >= 0 && value.int64 <= req->func->cpoolsz); return llvm_ptr(req, req->func->cpool + value.int64); case JIT_ADDR_REG: - assert(value.reg < req->func->nregs); - return LLVMBuildIntToPtr(req->builder, cgb->outregs[value.reg], - req->types[LLVM_PTR], ""); + { + assert(value.reg < req->func->nregs); + LLVMValueRef ptr = cgb->outregs[value.reg]; + + if (req->regtypes[value.reg] != LLVM_PTR) + ptr = LLVMBuildIntToPtr(req->builder, ptr, + req->types[LLVM_PTR], ""); + + if (value.disp != 0) { + LLVMValueRef indexes[] = { llvm_intptr(req, value.disp) }; + ptr = LLVMBuildGEP2(req->builder, req->types[LLVM_INT8], ptr, + indexes, ARRAY_LEN(indexes), ""); + } + + return ptr; + } /* case JIT_ADDR_ABS: return (jit_scalar_t){ .pointer = (void *)(intptr_t)value.int64 }; @@ -472,11 +502,35 @@ static LLVMValueRef cgen_get_value(cgen_req_t *req, cgen_block_t *cgb, case JIT_VALUE_EXIT: case JIT_VALUE_HANDLE: return llvm_int32(req, value.exit); + case JIT_ADDR_ABS: + return llvm_ptr(req, (void *)(intptr_t)value.int64); default: fatal_trace("cannot handle value kind %d", value.kind); } } +static LLVMValueRef cgen_coerce_value(cgen_req_t *req, cgen_block_t *cgb, + jit_value_t value, llvm_type_t type) +{ + LLVMValueRef raw = cgen_get_value(req, cgb, value); + + if (value.kind == JIT_VALUE_REG && req->regtypes[value.reg] == type) + return raw; + + LLVMTypeRef lltype = LLVMTypeOf(raw); + + switch (type) { + case LLVM_PTR: + if (LLVMGetTypeKind(lltype) == LLVMIntegerTypeKind) + return LLVMBuildIntToPtr(req->builder, raw, req->types[LLVM_PTR], ""); + else + return raw; + + default: + return raw; + } +} + static void cgen_coerece(cgen_req_t *req, jit_size_t size, LLVMValueRef *arg1, LLVMValueRef *arg2) { @@ -562,7 +616,7 @@ static void cgen_op_send(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) static void cgen_op_store(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) { LLVMValueRef value = cgen_get_value(req, cgb, ir->arg1); - LLVMValueRef ptr = cgen_get_value(req, cgb, ir->arg2); + LLVMValueRef ptr = cgen_coerce_value(req, cgb, ir->arg2, LLVM_PTR); LLVMValueRef trunc = value; switch (ir->size) { @@ -585,7 +639,7 @@ static void cgen_op_store(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) static void cgen_op_load(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) { - LLVMValueRef ptr = cgen_get_value(req, cgb, ir->arg1); + LLVMValueRef ptr = cgen_coerce_value(req, cgb, ir->arg1, LLVM_PTR); LLVMValueRef result = NULL; switch (ir->size) { @@ -833,8 +887,8 @@ static void cgen_op_neg(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) static void cgen_macro_copy(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) { LLVMValueRef count = cgb->outregs[ir->result]; - LLVMValueRef dest = cgen_get_value(req, cgb, ir->arg1); - LLVMValueRef src = cgen_get_value(req, cgb, ir->arg2); + LLVMValueRef dest = cgen_coerce_value(req, cgb, ir->arg1, LLVM_PTR); + LLVMValueRef src = cgen_coerce_value(req, cgb, ir->arg2, LLVM_PTR); LLVMBuildMemMove(req->builder, dest, 0, src, 0, count); } @@ -859,6 +913,26 @@ static void cgen_macro_exit(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) args, ARRAY_LEN(args), ""); } +static void cgen_macro_fficall(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) +{ + const unsigned irpos = ir - req->func->irbuf; + LLVMValueRef irpos_ptr = LLVMBuildStructGEP2(req->builder, + req->types[LLVM_ANCHOR], + req->anchor, 2, ""); + LLVMBuildStore(req->builder, llvm_int32(req, irpos), irpos_ptr); + + LLVMValueRef ff = cgen_get_value(req, cgb, ir->arg1); + LLVMValueRef fn = cgen_get_fn(req, LLVM_DO_FFICALL); + + LLVMValueRef args[] = { + ff, + req->anchor, + req->args + }; + LLVMBuildCall2(req->builder, req->fntypes[LLVM_DO_FFICALL], fn, + args, ARRAY_LEN(args), ""); +} + static void cgen_macro_galloc(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) { // TODO: use TLAB @@ -872,7 +946,8 @@ static void cgen_macro_galloc(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) }; cgb->outregs[ir->result] = LLVMBuildCall2(req->builder, req->fntypes[LLVM_MSPACE_ALLOC], - fn, args, ARRAY_LEN(args), ""); + fn, args, ARRAY_LEN(args), + cgen_reg_name(ir->result)); } static void cgen_macro_getpriv(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) @@ -965,6 +1040,9 @@ static void cgen_ir(cgen_req_t *req, cgen_block_t *cgb, jit_ir_t *ir) case MACRO_EXIT: cgen_macro_exit(req, cgb, ir); break; + case MACRO_FFICALL: + cgen_macro_fficall(req, cgb, ir); + break; case MACRO_GALLOC: cgen_macro_galloc(req, cgb, ir); break; @@ -1055,20 +1133,6 @@ static void cgen_hint_reg_size(cgen_req_t *req, jit_reg_t reg, jit_size_t sz) cgen_set_reg_type(req, reg, type); } -static void cgen_hint_value_size(cgen_req_t *req, jit_value_t value, - jit_size_t sz) -{ - switch (value.kind) { - case JIT_VALUE_REG: - case JIT_ADDR_REG: - cgen_hint_reg_size(req, value.reg, sz); - break; - - default: - break; - } -} - static void cgen_hint_reg_type(cgen_req_t *req, jit_reg_t reg, jit_value_t value) { @@ -1106,7 +1170,6 @@ static void cgen_reg_types(cgen_req_t *req) case J_ADD: case J_SUB: cgen_hint_reg_size(req, ir->result, ir->size); - cgen_hint_reg_type(req, ir->result, ir->arg1); break; case J_CSET: @@ -1115,7 +1178,6 @@ static void cgen_reg_types(cgen_req_t *req) case J_STORE: cgen_hint_value_type(req, ir->arg2, LLVM_PTR); - cgen_hint_value_size(req, ir->arg1, ir->size); break; case J_LEA: @@ -1146,6 +1208,7 @@ static void cgen_reg_types(cgen_req_t *req) case MACRO_EXIT: case MACRO_COPY: case MACRO_PUTPRIV: + case MACRO_FFICALL: break; default: diff --git a/src/jit/jit-priv.h b/src/jit/jit-priv.h index c67cdcf1..a0b08344 100644 --- a/src/jit/jit-priv.h +++ b/src/jit/jit-priv.h @@ -294,5 +294,7 @@ jit_block_t *jit_block_for(jit_cfg_t *cfg, int pos); int jit_get_edge(jit_edge_list_t *list, int nth); void __nvc_do_exit(jit_exit_t which, jit_anchor_t *anchor, jit_scalar_t *args); +void __nvc_do_fficall(jit_foreign_t *ff, jit_anchor_t *anchor, + jit_scalar_t *args); #endif // _JIT_PRIV_H diff --git a/src/symbols.txt b/src/symbols.txt index 3bc81e0e..124dd48b 100644 --- a/src/symbols.txt +++ b/src/symbols.txt @@ -36,6 +36,7 @@ __nvc_claim_tlab; __nvc_div_zero; __nvc_do_exit; + __nvc_do_fficall; __nvc_drive_signal; __nvc_elab_order_fail; __nvc_exponent_fail; -- 2.39.2