From e5f72a0de85067c342a8004bc26b577509dc3c50 Mon Sep 17 00:00:00 2001 From: Nick Gasson Date: Sun, 4 Dec 2022 21:45:33 +0000 Subject: [PATCH] Implement TLAB in interpreter and LLVM JIT --- src/jit/jit-core.c | 8 +-- src/jit/jit-dump.c | 3 +- src/jit/jit-exits.c | 25 ++++++- src/jit/jit-exits.h | 2 +- src/jit/jit-interp.c | 29 +++++++-- src/jit/jit-irgen.c | 15 ++++- src/jit/jit-llvm.c | 152 +++++++++++++++++++++++++++++++++++++++---- src/jit/jit-priv.h | 11 +++- src/jit/jit.h | 2 +- src/rt/model.c | 18 ++--- src/symbols.txt | 1 + test/jitperf.c | 2 +- test/test_jit.c | 6 +- 13 files changed, 232 insertions(+), 42 deletions(-) diff --git a/src/jit/jit-core.c b/src/jit/jit-core.c index 73a9cd64..1d4d801e 100644 --- a/src/jit/jit-core.c +++ b/src/jit/jit-core.c @@ -397,7 +397,7 @@ void *jit_link(jit_t *j, jit_handle_t handle) fatal_trace("cannot link unit %s", istr(f->name)); jit_scalar_t p1 = { .pointer = NULL }, p2 = p1, result; - if (!jit_fastcall(j, f->handle, &result, p1, p2)) { + if (!jit_fastcall(j, f->handle, &result, p1, p2, NULL)) { error_at(&(f->object->loc), "failed to initialise %s", istr(f->name)); result.pointer = NULL; } @@ -592,7 +592,7 @@ static void jit_transition(jit_t *j, jit_state_t from, jit_state_t to) } bool jit_fastcall(jit_t *j, jit_handle_t handle, jit_scalar_t *result, - jit_scalar_t p1, jit_scalar_t p2) + jit_scalar_t p1, jit_scalar_t p2, tlab_t *tlab) { jit_func_t *f = jit_get_func(j, handle); jit_thread_local_t *thread = jit_thread_local(); @@ -613,7 +613,7 @@ bool jit_fastcall(jit_t *j, jit_handle_t handle, jit_scalar_t *result, jit_scalar_t args[JIT_MAX_ARGS]; args[0] = p1; args[1] = p2; - (*f->entry)(f, NULL, args); + (*f->entry)(f, NULL, args, tlab); *result = args[0]; jit_transition(j, JIT_INTERP, JIT_IDLE); } @@ -652,7 +652,7 @@ static bool jit_try_vcall(jit_t *j, jit_func_t *f, jit_scalar_t *result, jit_ffi_call(ff, args); } else - (*f->entry)(f, NULL, args); + (*f->entry)(f, NULL, args, NULL); *result = args[0]; } diff --git a/src/jit/jit-dump.c b/src/jit/jit-dump.c index 093ae0bd..61dc6873 100644 --- a/src/jit/jit-dump.c +++ b/src/jit/jit-dump.c @@ -41,7 +41,7 @@ const char *jit_op_name(jit_op_t op) if (op >= __MACRO_BASE) { static const char *names[] = { "$COPY", "$GALLOC", "$EXIT", "$FEXP", "$EXP", "$BZERO", - "$FFICALL", "$GETPRIV", "$PUTPRIV", + "$FFICALL", "$GETPRIV", "$PUTPRIV", "$LALLOC", }; assert(op - __MACRO_BASE < ARRAY_LEN(names)); return names[op - __MACRO_BASE]; @@ -71,6 +71,7 @@ const char *jit_exit_name(jit_exit_t exit) "MAP_CONST", "RESOLVE_SIGNAL", "LAST_EVENT", "LAST_ACTIVE", "DISCONNECT", "ELAB_ORDER_FAIL", "FORCE", "RELEASE", "PUSH_SCOPE", "POP_SCOPE", "IMPLICIT_SIGNAL", "DRIVING", "DRIVING_VALUE", + "CLAIM_TLAB", }; assert(exit < ARRAY_LEN(names)); return names[exit]; diff --git a/src/jit/jit-exits.c b/src/jit/jit-exits.c index e3e49134..4407fbb5 100644 --- a/src/jit/jit-exits.c +++ b/src/jit/jit-exits.c @@ -532,7 +532,8 @@ void x_func_wait(void) // Entry point from interpreter or JIT compiled code DLLEXPORT -void __nvc_do_exit(jit_exit_t which, jit_anchor_t *anchor, jit_scalar_t *args) +void __nvc_do_exit(jit_exit_t which, jit_anchor_t *anchor, jit_scalar_t *args, + tlab_t *tlab) { jit_thread_local_t *thread = jit_thread_local(); thread->anchor = anchor; @@ -1052,6 +1053,13 @@ void __nvc_do_exit(jit_exit_t which, jit_anchor_t *anchor, jit_scalar_t *args) } break; + case JIT_EXIT_CLAIM_TLAB: + { + if (tlab != NULL) + x_claim_tlab(tlab); + } + break; + default: fatal_trace("unhandled exit %s", jit_exit_name(which)); } @@ -1291,7 +1299,8 @@ void __nvc_report(const uint8_t *msg, int32_t msg_len, int8_t severity, DLLEXPORT void __nvc_claim_tlab(void) { - x_claim_tlab(); + extern tlab_t __nvc_tlab; + x_claim_tlab(&__nvc_tlab); } DLLEXPORT @@ -1436,6 +1445,18 @@ void *__nvc_mspace_alloc(uint32_t size, uint32_t nelems) return x_mspace_alloc(size, nelems); } +DLLEXPORT +void *__nvc_mspace_alloc2(uintptr_t size, jit_anchor_t *anchor) +{ + jit_thread_local_t *thread = jit_thread_local(); + thread->anchor = anchor; + + void *ptr = x_mspace_alloc(size, 1); + + thread->anchor = NULL; + return ptr; +} + DLLEXPORT jit_handle_t __nvc_get_handle(const char *func, ffi_spec_t spec) { diff --git a/src/jit/jit-exits.h b/src/jit/jit-exits.h index d03c7983..ead965d6 100644 --- a/src/jit/jit-exits.h +++ b/src/jit/jit-exits.h @@ -68,7 +68,7 @@ void x_assert_fail(const uint8_t *msg, int32_t msg_len, int8_t severity, tree_t where); void x_report(const uint8_t *msg, int32_t msg_len, int8_t severity, tree_t where); -void x_claim_tlab(void); +void x_claim_tlab(tlab_t *tlab); int64_t x_last_event(sig_shared_t *ss, uint32_t offset, int32_t count); int64_t x_last_active(sig_shared_t *ss, uint32_t offset, int32_t count); void x_map_signal(sig_shared_t *src_ss, uint32_t src_offset, diff --git a/src/jit/jit-interp.c b/src/jit/jit-interp.c index 41d8ccc9..72799b69 100644 --- a/src/jit/jit-interp.c +++ b/src/jit/jit-interp.c @@ -45,6 +45,7 @@ typedef struct _jit_interp { mspace_t *mspace; unsigned backedge; jit_anchor_t *anchor; + tlab_t *tlab; } jit_interp_t; #ifdef DEBUG @@ -567,7 +568,7 @@ static void interp_call(jit_interp_t *state, jit_ir_t *ir) jit_msg(NULL, DIAG_FATAL, "missing definition for subprogram"); else { jit_func_t *f = jit_get_func(state->func->jit, ir->arg1.handle); - (*f->entry)(f, state->anchor, state->args); + (*f->entry)(f, state->anchor, state->args, state->tlab); } } @@ -668,10 +669,25 @@ static void interp_galloc(jit_interp_t *state, jit_ir_t *ir) thread->anchor = NULL; } +static void interp_lalloc(jit_interp_t *state, jit_ir_t *ir) +{ + jit_thread_local_t *thread = jit_thread_local(); + thread->anchor = state->anchor; + + const size_t bytes = interp_get_value(state, ir->arg1).integer; + + if (state->tlab != NULL) + state->regs[ir->result].pointer = tlab_alloc(state->tlab, bytes); + else + state->regs[ir->result].pointer = mspace_alloc(state->mspace, bytes); + + thread->anchor = NULL; +} + static void interp_exit(jit_interp_t *state, jit_ir_t *ir) { state->anchor->irpos = ir - state->func->irbuf; - __nvc_do_exit(ir->arg1.exit, state->anchor, state->args); + __nvc_do_exit(ir->arg1.exit, state->anchor, state->args, state->tlab); } static void interp_fficall(jit_interp_t *state, jit_ir_t *ir) @@ -809,6 +825,9 @@ static void interp_loop(jit_interp_t *state) case MACRO_GALLOC: interp_galloc(state, ir); break; + case MACRO_LALLOC: + interp_lalloc(state, ir); + break; case MACRO_EXIT: interp_exit(state, ir); break; @@ -834,12 +853,13 @@ static void interp_loop(jit_interp_t *state) } } -void jit_interp(jit_func_t *f, jit_anchor_t *caller, jit_scalar_t *args) +void jit_interp(jit_func_t *f, jit_anchor_t *caller, jit_scalar_t *args, + tlab_t *tlab) { if (f->entry != jit_interp) { // Came from stale compiled code // TODO: should we patch the call site? - (*f->entry)(f, caller, args); + (*f->entry)(f, caller, args, tlab); return; } @@ -874,6 +894,7 @@ void jit_interp(jit_func_t *f, jit_anchor_t *caller, jit_scalar_t *args) .mspace = jit_get_mspace(f->jit), .backedge = jit_backedge_limit(f->jit), .anchor = &anchor, + .tlab = tlab, }; interp_loop(&state); diff --git a/src/jit/jit-irgen.c b/src/jit/jit-irgen.c index b2cebbf2..4f722851 100644 --- a/src/jit/jit-irgen.c +++ b/src/jit/jit-irgen.c @@ -567,6 +567,13 @@ static jit_value_t macro_galloc(jit_irgen_t *g, jit_value_t bytes) return jit_value_from_reg(r); } +static jit_value_t macro_lalloc(jit_irgen_t *g, jit_value_t bytes) +{ + jit_reg_t r = irgen_alloc_reg(g); + irgen_emit_unary(g, MACRO_LALLOC, JIT_SZ_UNSPEC, JIT_CC_NONE, r, bytes); + return jit_value_from_reg(r); +} + static void macro_exit(jit_irgen_t *g, jit_exit_t exit) { irgen_emit_unary(g, MACRO_EXIT, JIT_SZ_UNSPEC, JIT_CC_NONE, JIT_REG_INVALID, @@ -2153,8 +2160,10 @@ static void irgen_op_wait(jit_irgen_t *g, int op) j_store(g, JIT_SZ_32, jit_value_from_int64(target), ptr); } - if (vcode_unit_kind() == VCODE_UNIT_PROCEDURE) + if (vcode_unit_kind() == VCODE_UNIT_PROCEDURE) { + macro_exit(g, JIT_EXIT_CLAIM_TLAB); j_send(g, 0, g->statereg); + } else j_send(g, 0, jit_value_from_int64(0)); @@ -2438,7 +2447,7 @@ static void irgen_op_alloc(jit_irgen_t *g, int op) jit_value_t count = irgen_get_arg(g, op, 0); jit_value_t total = j_mul(g, count, jit_value_from_int64(bytes)); - g->map[vcode_get_result(op)] = macro_galloc(g, total); + g->map[vcode_get_result(op)] = macro_lalloc(g, total); } static void irgen_op_all(jit_irgen_t *g, int op) @@ -3549,7 +3558,7 @@ static void irgen_locals(jit_irgen_t *g) sz += irgen_size_bytes(vtype); } - jit_value_t mem = macro_galloc(g, jit_value_from_int64(sz)); + jit_value_t mem = macro_lalloc(g, jit_value_from_int64(sz)); if (g->statereg.kind != JIT_VALUE_INVALID) { // A null state was passed in by the caller j_mov(g, jit_value_as_reg(g->statereg), mem); diff --git a/src/jit/jit-llvm.c b/src/jit/jit-llvm.c index 2b47dd3f..47f2f0fb 100644 --- a/src/jit/jit-llvm.c +++ b/src/jit/jit-llvm.c @@ -66,6 +66,7 @@ typedef enum { LLVM_ANCHOR, LLVM_CTOR_FN, LLVM_CTOR, + LLVM_TLAB, LLVM_LAST_TYPE } llvm_type_t; @@ -115,6 +116,7 @@ typedef enum { LLVM_GET_TREE, LLVM_GET_OBJECT, LLVM_GET_HANDLE, + LLVM_TLAB_ALLOC, LLVM_LAST_FN, } llvm_fn_t; @@ -153,6 +155,7 @@ typedef struct _cgen_block { typedef struct _cgen_func { LLVMValueRef llvmfn; LLVMValueRef args; + LLVMValueRef tlab; LLVMValueRef frame; LLVMValueRef anchor; LLVMValueRef cpool; @@ -241,7 +244,8 @@ static void llvm_register_types(llvm_obj_t *obj) LLVMTypeRef atypes[] = { obj->types[LLVM_PTR], // Function obj->types[LLVM_PTR], // Anchor - obj->types[LLVM_PTR] // Arguments + obj->types[LLVM_PTR], // Arguments + obj->types[LLVM_PTR] // TLAB pointer }; obj->types[LLVM_ENTRY_FN] = LLVMFunctionType(obj->types[LLVM_VOID], atypes, ARRAY_LEN(atypes), @@ -262,6 +266,18 @@ static void llvm_register_types(llvm_obj_t *obj) false); } + { + LLVMTypeRef fields[] = { + obj->types[LLVM_PTR], // Mspace object + obj->types[LLVM_PTR], // Base pointer + obj->types[LLVM_PTR], // Allocation pointer + obj->types[LLVM_PTR], // Limit pointer + obj->types[LLVM_INT32], // Mptr object + }; + obj->types[LLVM_TLAB] = LLVMStructTypeInContext(obj->context, fields, + ARRAY_LEN(fields), false); + } + { LLVMTypeRef fields[] = { obj->types[LLVM_INT32], @@ -548,7 +564,8 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) LLVMTypeRef args[] = { obj->types[LLVM_INT32], obj->types[LLVM_PTR], - obj->types[LLVM_PTR] + obj->types[LLVM_PTR], + obj->types[LLVM_PTR], }; obj->fntypes[which] = LLVMFunctionType(obj->types[LLVM_VOID], args, ARRAY_LEN(args), false); @@ -597,13 +614,13 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) case LLVM_MSPACE_ALLOC: { LLVMTypeRef args[] = { - obj->types[LLVM_INT32], - obj->types[LLVM_INT32] + obj->types[LLVM_INTPTR], + obj->types[LLVM_PTR] }; obj->fntypes[which] = LLVMFunctionType(obj->types[LLVM_PTR], args, ARRAY_LEN(args), false); - fn = llvm_add_fn(obj, "__nvc_mspace_alloc", obj->fntypes[which]); + fn = llvm_add_fn(obj, "__nvc_mspace_alloc2", obj->fntypes[which]); } break; @@ -680,6 +697,19 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) } break; + case LLVM_TLAB_ALLOC: + { + LLVMTypeRef args[] = { + obj->types[LLVM_PTR], + obj->types[LLVM_INTPTR], + obj->types[LLVM_PTR] + }; + obj->fntypes[which] = LLVMFunctionType(obj->types[LLVM_PTR], args, + ARRAY_LEN(args), false); + fn = llvm_add_fn(obj, "tlab_alloc", obj->fntypes[which]); + } + break; + default: fatal_trace("cannot generate prototype for function %d", which); } @@ -1509,7 +1539,8 @@ static void cgen_op_call(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) LLVMValueRef args[] = { fptr, PTR(cgb->func->anchor), - cgb->func->args + cgb->func->args, + cgb->func->tlab, }; LLVMBuildCall2(obj->builder, obj->types[LLVM_ENTRY_FN], entry, args, ARRAY_LEN(args), ""); @@ -1595,7 +1626,8 @@ static void cgen_macro_exit(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) LLVMValueRef args[] = { which, PTR(cgb->func->anchor), - cgb->func->args + cgb->func->args, + cgb->func->tlab, }; llvm_call_fn(obj, LLVM_DO_EXIT, args, ARRAY_LEN(args)); } @@ -1616,15 +1648,13 @@ static void cgen_macro_fficall(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) static void cgen_macro_galloc(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { - // TODO: use TLAB - cgen_sync_irpos(obj, cgb, ir); LLVMValueRef size = cgen_get_value(obj, cgb, ir->arg1); LLVMValueRef args[] = { - LLVMBuildTrunc(obj->builder, size, obj->types[LLVM_INT32], ""), - llvm_int32(obj, 1), + LLVMBuildTrunc(obj->builder, size, obj->types[LLVM_INTPTR], ""), + cgb->func->anchor }; LLVMValueRef ptr = llvm_call_fn(obj, LLVM_MSPACE_ALLOC, args, ARRAY_LEN(args)); @@ -1634,6 +1664,25 @@ static void cgen_macro_galloc(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) cgen_reg_name(ir->result)); } +static void cgen_macro_lalloc(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) +{ + cgen_sync_irpos(obj, cgb, ir); + + LLVMValueRef size = cgen_get_value(obj, cgb, ir->arg1); + + LLVMValueRef args[] = { + cgb->func->tlab, + LLVMBuildTrunc(obj->builder, size, obj->types[LLVM_INTPTR], ""), + cgb->func->anchor + }; + LLVMValueRef ptr = llvm_call_fn(obj, LLVM_TLAB_ALLOC, args, + ARRAY_LEN(args)); + + cgb->outregs[ir->result] = LLVMBuildPtrToInt(obj->builder, ptr, + obj->types[LLVM_INT64], + cgen_reg_name(ir->result)); +} + static void cgen_macro_getpriv(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { // TODO: this needs some kind of fast-path @@ -1775,6 +1824,9 @@ static void cgen_ir(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) case MACRO_GALLOC: cgen_macro_galloc(obj, cgb, ir); break; + case MACRO_LALLOC: + cgen_macro_lalloc(obj, cgb, ir); + break; case MACRO_GETPRIV: cgen_macro_getpriv(obj, cgb, ir); break; @@ -2014,6 +2066,9 @@ static void cgen_function(llvm_obj_t *obj, cgen_func_t *func) func->args = LLVMGetParam(func->llvmfn, 2); LLVMSetValueName(func->args, "args"); + func->tlab = LLVMGetParam(func->llvmfn, 3); + LLVMSetValueName(func->tlab, "tlab"); + if (func->source->framesz > 0) { LLVMTypeRef frame_type = LLVMArrayType(obj->types[LLVM_INT8], func->source->framesz); @@ -2118,6 +2173,75 @@ static void cgen_function(llvm_obj_t *obj, cgen_func_t *func) func->blocks = NULL; } +static void cgen_tlab_alloc_body(llvm_obj_t *obj) +{ + LLVMValueRef fn = obj->fns[LLVM_TLAB_ALLOC]; + LLVMSetLinkage(fn, LLVMPrivateLinkage); + + LLVMBasicBlockRef entry = llvm_append_block(obj, fn, ""); + + LLVMPositionBuilderAtEnd(obj->builder, entry); + + LLVMValueRef tlab = LLVMGetParam(fn, 0); + LLVMSetValueName(tlab, "tlab"); + + LLVMValueRef bytes = LLVMGetParam(fn, 1); + LLVMSetValueName(bytes, "bytes"); + + LLVMValueRef anchor = LLVMGetParam(fn, 2); + LLVMSetValueName(anchor, "anchor"); + + LLVMValueRef tlab_valid = LLVMBuildIsNotNull(obj->builder, tlab, ""); + + LLVMBasicBlockRef fast_bb = llvm_append_block(obj, fn, ""); + LLVMBasicBlockRef update_bb = llvm_append_block(obj, fn, ""); + LLVMBasicBlockRef slow_bb = llvm_append_block(obj, fn, ""); + + LLVMBuildCondBr(obj->builder, tlab_valid, fast_bb, slow_bb); + + LLVMPositionBuilderAtEnd(obj->builder, fast_bb); + + LLVMValueRef alloc_ptr = + LLVMBuildStructGEP2(obj->builder, obj->types[LLVM_TLAB], tlab, 2, ""); + LLVMValueRef limit_ptr = + LLVMBuildStructGEP2(obj->builder, obj->types[LLVM_TLAB], tlab, 3, ""); + + LLVMValueRef alloc = + LLVMBuildLoad2(obj->builder, obj->types[LLVM_PTR], alloc_ptr, ""); + LLVMValueRef limit = + LLVMBuildLoad2(obj->builder, obj->types[LLVM_PTR], limit_ptr, ""); + + LLVMValueRef align_mask = llvm_intptr(obj, RT_ALIGN_MASK); + LLVMValueRef align_up = + LLVMBuildAnd(obj->builder, + LLVMBuildAdd(obj->builder, bytes, align_mask, ""), + LLVMBuildNot(obj->builder, align_mask, ""), ""); + + LLVMValueRef indexes[] = { + LLVMBuildZExt(obj->builder, align_up, obj->types[LLVM_INTPTR], "") + }; + LLVMValueRef next = LLVMBuildInBoundsGEP2(obj->builder, + obj->types[LLVM_INT8], + alloc, indexes, + ARRAY_LEN(indexes), ""); + + LLVMValueRef over = LLVMBuildICmp(obj->builder, LLVMIntUGT, next, limit, ""); + LLVMBuildCondBr(obj->builder, over, slow_bb, update_bb); + + LLVMPositionBuilderAtEnd(obj->builder, update_bb); + + LLVMBuildStore(obj->builder, next, alloc_ptr); + LLVMBuildRet(obj->builder, alloc); + + LLVMPositionBuilderAtEnd(obj->builder, slow_bb); + + LLVMValueRef args[] = { bytes, anchor }; + LLVMValueRef ptr = llvm_call_fn(obj, LLVM_MSPACE_ALLOC, args, + ARRAY_LEN(args)); + + LLVMBuildRet(obj->builder, ptr); +} + //////////////////////////////////////////////////////////////////////////////// // JIT plugin interface @@ -2193,6 +2317,9 @@ static void jit_llvm_cgen(jit_t *j, jit_handle_t handle, void *context) cgen_function(&obj, &func); + if (obj.fns[LLVM_TLAB_ALLOC] != NULL) + cgen_tlab_alloc_body(&obj); + llvm_finalise(&obj); LLVMOrcThreadSafeModuleRef tsm = @@ -2353,6 +2480,9 @@ void llvm_obj_emit(llvm_obj_t *obj, const char *path) LLVMBuildRetVoid(obj->builder); } + if (obj->fns[LLVM_TLAB_ALLOC] != NULL) + cgen_tlab_alloc_body(obj); + llvm_finalise(obj); char *error; diff --git a/src/jit/jit-priv.h b/src/jit/jit-priv.h index 270c7461..d63432d4 100644 --- a/src/jit/jit-priv.h +++ b/src/jit/jit-priv.h @@ -74,6 +74,7 @@ typedef enum { MACRO_FFICALL, MACRO_GETPRIV, MACRO_PUTPRIV, + MACRO_LALLOC, } jit_op_t; typedef enum { @@ -146,6 +147,7 @@ typedef enum { JIT_EXIT_IMPLICIT_SIGNAL, JIT_EXIT_DRIVING, JIT_EXIT_DRIVING_VALUE, + JIT_EXIT_CLAIM_TLAB, } jit_exit_t; typedef uint16_t jit_reg_t; @@ -206,7 +208,8 @@ typedef struct _jit_func jit_func_t; typedef struct _jit_block jit_block_t; typedef struct _jit_anchor jit_anchor_t; -typedef void (*jit_entry_fn_t)(jit_func_t *, jit_anchor_t *, jit_scalar_t *); +typedef void (*jit_entry_fn_t)(jit_func_t *, jit_anchor_t *, + jit_scalar_t *, tlab_t *); typedef struct { unsigned count; @@ -304,7 +307,8 @@ void jit_dump_with_mark(jit_func_t *f, jit_label_t label, bool cpool); void jit_dump_interleaved(jit_func_t *f); const char *jit_op_name(jit_op_t op); const char *jit_exit_name(jit_exit_t exit); -void jit_interp(jit_func_t *f, jit_anchor_t *caller, jit_scalar_t *args); +void jit_interp(jit_func_t *f, jit_anchor_t *caller, jit_scalar_t *args, + tlab_t *tlab); jit_func_t *jit_get_func(jit_t *j, jit_handle_t handle); void jit_hexdump(const unsigned char *data, size_t sz, int blocksz, const void *highlight, const char *prefix); @@ -325,7 +329,8 @@ int jit_get_edge(jit_edge_list_t *list, int nth); void jit_do_lvn(jit_func_t *f); -void __nvc_do_exit(jit_exit_t which, jit_anchor_t *anchor, jit_scalar_t *args); +void __nvc_do_exit(jit_exit_t which, jit_anchor_t *anchor, jit_scalar_t *args, + tlab_t *tlab); void __nvc_do_fficall(jit_foreign_t *ff, jit_anchor_t *anchor, jit_scalar_t *args); diff --git a/src/jit/jit.h b/src/jit/jit.h index e0d8b84e..c1d41abf 100644 --- a/src/jit/jit.h +++ b/src/jit/jit.h @@ -78,7 +78,7 @@ bool jit_try_call_packed(jit_t *j, jit_handle_t handle, jit_scalar_t context, jit_scalar_t jit_call(jit_t *j, jit_handle_t handle, ...); bool jit_call_thunk(jit_t *j, vcode_unit_t unit, jit_scalar_t *result); bool jit_fastcall(jit_t *j, jit_handle_t handle, jit_scalar_t *result, - jit_scalar_t p1, jit_scalar_t p2); + jit_scalar_t p1, jit_scalar_t p2, tlab_t *tlab); __attribute__((format(printf, 3, 4))) void jit_msg(const loc_t *where, diag_level_t level, const char *fmt, ...); diff --git a/src/rt/model.c b/src/rt/model.c index 1cdae8fe..941122be 100644 --- a/src/rt/model.c +++ b/src/rt/model.c @@ -793,7 +793,7 @@ static void reset_process(rt_model_t *m, rt_proc_t *proc) jit_scalar_t state = { .pointer = NULL }; jit_scalar_t result; - if (jit_fastcall(m->jit, proc->handle, &result, state, context)) + if (jit_fastcall(m->jit, proc->handle, &result, state, context, NULL)) mptr_put(m->mspace, proc->privdata, result.pointer); else m->force_stop = true; @@ -815,6 +815,8 @@ static void run_process(rt_model_t *m, rt_proc_t *proc) else if (!tlab_valid(__nvc_tlab)) tlab_acquire(m->mspace, &__nvc_tlab); + tlab_t *tlab = &__nvc_tlab; + active_proc = proc; active_scope = proc->scope; @@ -829,7 +831,7 @@ static void run_process(rt_model_t *m, rt_proc_t *proc) .pointer = mptr_get(m->mspace, proc->scope->privdata) }; - if (!jit_fastcall(m->jit, proc->handle, &result, state, context)) + if (!jit_fastcall(m->jit, proc->handle, &result, state, context, tlab)) m->force_stop = true; active_proc = NULL; @@ -870,7 +872,7 @@ static void reset_scope(rt_model_t *m, rt_scope_t *s) if (s->parent != NULL) context.pointer = mptr_get(m->mspace, s->parent->privdata); - if (jit_fastcall(m->jit, handle, &result, context, p2)) + if (jit_fastcall(m->jit, handle, &result, context, p2, NULL)) mptr_put(m->mspace, s->privdata, result.pointer); else { m->force_stop = true; @@ -2906,15 +2908,15 @@ void x_alias_signal(sig_shared_t *ss, tree_t where) active_scope->aliases = a; } -void x_claim_tlab(void) +void x_claim_tlab(tlab_t *tlab) { TRACE("claiming TLAB for private use (used %zu/%d)", - __nvc_tlab.alloc - __nvc_tlab.base, TLAB_SIZE); + tlab->alloc - tlab->base, TLAB_SIZE); - assert(tlab_valid(__nvc_tlab)); - assert(__nvc_tlab.alloc > __nvc_tlab.base); + assert(tlab_valid(*tlab)); + assert(tlab->alloc > tlab->base); - tlab_move(__nvc_tlab, active_proc->tlab); + tlab_move(*tlab, active_proc->tlab); } int64_t x_last_event(sig_shared_t *ss, uint32_t offset, int32_t count) diff --git a/src/symbols.txt b/src/symbols.txt index 92086d33..bf3372b0 100644 --- a/src/symbols.txt +++ b/src/symbols.txt @@ -53,6 +53,7 @@ __nvc_map_const; __nvc_map_signal; __nvc_mspace_alloc; + __nvc_mspace_alloc2; __nvc_null_deref; __nvc_overflow; __nvc_pop_scope; diff --git a/test/jitperf.c b/test/jitperf.c index 9385983d..bcf3b9cc 100644 --- a/test/jitperf.c +++ b/test/jitperf.c @@ -84,7 +84,7 @@ static void run_benchmark(tree_t pack, tree_t proc) for (; (now = get_timestamp_us()) < start + 1000000; iters++) { jit_scalar_t result; jit_scalar_t dummy = { .integer = 0 }; - if (!jit_fastcall(j, hproc, &result, context, dummy)) + if (!jit_fastcall(j, hproc, &result, context, dummy, NULL)) fatal("error in benchmark subprogram"); } diff --git a/test/test_jit.c b/test/test_jit.c index 502e6abc..4a038715 100644 --- a/test/test_jit.c +++ b/test/test_jit.c @@ -1180,7 +1180,7 @@ START_TEST(test_assemble1) jit_handle_t h1 = jit_assemble(j, ident_new("myfunc"), text1); jit_scalar_t result, p0 = { .integer = 5 }; - fail_unless(jit_fastcall(j, h1, &result, p0, p0)); + fail_unless(jit_fastcall(j, h1, &result, p0, p0, NULL)); ck_assert_int_eq(result.integer, 7); @@ -1204,7 +1204,7 @@ START_TEST(test_assemble2) jit_handle_t h1 = jit_assemble(j, ident_new("myfunc"), text1); jit_scalar_t result, p0 = { .integer = 5 }; - fail_unless(jit_fastcall(j, h1, &result, p0, p0)); + fail_unless(jit_fastcall(j, h1, &result, p0, p0, NULL)); ck_assert_int_eq(result.integer, 0); @@ -1230,7 +1230,7 @@ START_TEST(test_cfg1) jit_handle_t h1 = jit_assemble(j, ident_new("myfunc"), text1); jit_scalar_t result, p0 = { .integer = 5 }; - fail_unless(jit_fastcall(j, h1, &result, p0, p0)); + fail_unless(jit_fastcall(j, h1, &result, p0, p0, NULL)); ck_assert_int_eq(result.integer, 120); -- 2.39.2