From 53217747e6bffa9072ee7074327707789c6137b7 Mon Sep 17 00:00:00 2001 From: Nick Gasson Date: Wed, 11 Jan 2023 22:09:11 +0000 Subject: [PATCH] Implement coverage ops in JIT --- src/jit/jit-core.c | 62 +++++++++++++++++++++++++++++++ src/jit/jit-dump.c | 10 ++++- src/jit/jit-exits.c | 11 +++++- src/jit/jit-interp.c | 4 ++ src/jit/jit-irgen.c | 87 ++++++++++++++++++++++++++++++++++++++++++++ src/jit/jit-llvm.c | 32 +++++++++++++++- src/jit/jit-pack.c | 4 +- src/jit/jit-priv.h | 6 ++- src/jit/jit-x86.c | 6 +++ src/jit/jit.h | 11 ++++++ src/nvc.c | 3 ++ src/rt/model.c | 28 +++++--------- test/run_regr.c | 1 - 13 files changed, 238 insertions(+), 27 deletions(-) diff --git a/src/jit/jit-core.c b/src/jit/jit-core.c index 77684482..7d5ab15d 100644 --- a/src/jit/jit-core.c +++ b/src/jit/jit-core.c @@ -101,6 +101,7 @@ typedef struct _jit { func_array_t *funcs; unsigned next_handle; nvc_lock_t lock; + int32_t *cover_mem[4]; } jit_t; static void jit_oom_cb(mspace_t *m, size_t size) @@ -318,6 +319,20 @@ static jit_handle_t jit_lazy_compile_locked(jit_t *j, ident_t name) ident_t id = ident_new(eptr + 1); r->ptr = jit_ffi_get(id) ?: jit_ffi_bind(id, spec, NULL); } + else if (r->kind == RELOC_COVER) { + // TODO: get rid of the double indirection here by + // allocating coverage memory earlier + if (strcmp(str, "stmt") == 0) + r->ptr = &(j->cover_mem[JIT_COVER_STMT]); + else if (strcmp(str, "branch") == 0) + r->ptr = &(j->cover_mem[JIT_COVER_BRANCH]); + else if (strcmp(str, "expr") == 0) + r->ptr = &(j->cover_mem[JIT_COVER_EXPRESSION]); + else if (strcmp(str, "toggle") == 0) + r->ptr = &(j->cover_mem[JIT_COVER_TOGGLE]); + else + fatal_trace("relocation against invalid coverage kind %s", str); + } else { jit_handle_t h = jit_lazy_compile_locked(j, ident_new(str)); if (h == JIT_HANDLE_INVALID) @@ -1416,3 +1431,50 @@ bool jit_will_abort(jit_ir_t *ir) else return ir->op == J_TRAP; } + +void jit_alloc_cover_mem(jit_t *j, int n_stmts, int n_branches, int n_toggles, + int n_expressions) +{ + int32_t *cover_stmts = ffi_find_symbol(NULL, "cover_stmts"); + if (cover_stmts != NULL) + memset(cover_stmts, '\0', sizeof(int32_t) * n_stmts); + else + cover_stmts = xcalloc_array(n_stmts, sizeof(int32_t)); // XXX: leaks + + int32_t *cover_branches = ffi_find_symbol(NULL, "cover_branches"); + if (cover_branches != NULL) + memset(cover_branches, '\0', sizeof(int32_t) * n_branches); + else + cover_branches = xcalloc_array(n_branches, sizeof(int32_t)); // XXX: leaks + + int32_t *cover_toggles = ffi_find_symbol(NULL, "cover_toggles"); + if (cover_toggles != NULL) + memset(cover_toggles, '\0', sizeof(int32_t) * n_toggles); + else + cover_toggles = xcalloc_array(n_toggles, sizeof(int32_t)); // XXX: leaks + + int32_t *cover_expressions = ffi_find_symbol(NULL, "cover_expressions"); + if (cover_expressions != NULL) + memset(cover_expressions, '\0', sizeof(int32_t) * n_expressions); + else + cover_expressions = xcalloc_array(n_expressions, sizeof(int32_t)); // XXX: leaks + + j->cover_mem[JIT_COVER_STMT] = cover_stmts; + j->cover_mem[JIT_COVER_BRANCH] = cover_branches; + j->cover_mem[JIT_COVER_TOGGLE] = cover_toggles; + j->cover_mem[JIT_COVER_EXPRESSION] = cover_expressions; +} + +int32_t *jit_get_cover_mem(jit_t *j, jit_cover_mem_t kind) +{ + assert(kind < ARRAY_LEN(j->cover_mem)); + return j->cover_mem[kind]; +} + +int32_t *jit_get_cover_ptr(jit_t *j, jit_value_t addr) +{ + assert(addr.kind == JIT_ADDR_COVER); + int32_t *base = jit_get_cover_mem(j, addr.int64 & 3); + assert(base != NULL); + return base + (addr.int64 >> 2); +} diff --git a/src/jit/jit-dump.c b/src/jit/jit-dump.c index 719673cd..8b654982 100644 --- a/src/jit/jit-dump.c +++ b/src/jit/jit-dump.c @@ -1,5 +1,5 @@ // -// Copyright (C) 2022 Nick Gasson +// Copyright (C) 2022-2023 Nick Gasson // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -73,7 +73,7 @@ const char *jit_exit_name(jit_exit_t exit) "MAP_CONST", "RESOLVE_SIGNAL", "LAST_EVENT", "LAST_ACTIVE", "DISCONNECT", "ELAB_ORDER_FAIL", "FORCE", "RELEASE", "PUSH_SCOPE", "POP_SCOPE", "IMPLICIT_SIGNAL", "DRIVING", "DRIVING_VALUE", - "CLAIM_TLAB", + "CLAIM_TLAB", "COVER_TOGGLE", }; assert(exit < ARRAY_LEN(names)); return names[exit]; @@ -112,6 +112,12 @@ static int jit_dump_value(jit_dump_t *d, jit_value_t value) } case JIT_ADDR_ABS: return printf("[#%016"PRIx64"]", value.int64); + case JIT_ADDR_COVER: + { + const char *kind[] = { "STMT", "BRANCH", "TOGGLE", "EXPR" }; + return printf("[$%s:%"PRIi64"]", kind[value.int64 & 3], + value.int64 >> 2); + } case JIT_VALUE_LABEL: if (value.label == JIT_LABEL_INVALID) return printf("???"); diff --git a/src/jit/jit-exits.c b/src/jit/jit-exits.c index 243a1183..b6b6b246 100644 --- a/src/jit/jit-exits.c +++ b/src/jit/jit-exits.c @@ -1,5 +1,5 @@ // -// Copyright (C) 2022 Nick Gasson +// Copyright (C) 2022-2023 Nick Gasson // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -1060,6 +1060,15 @@ void __nvc_do_exit(jit_exit_t which, jit_anchor_t *anchor, jit_scalar_t *args, } break; + case JIT_EXIT_COVER_TOGGLE: + { + sig_shared_t *shared = args[0].pointer; + int32_t *mem = args[1].pointer; + + x_cover_setup_toggle_cb(shared, mem); + } + break; + default: fatal_trace("unhandled exit %s", jit_exit_name(which)); } diff --git a/src/jit/jit-interp.c b/src/jit/jit-interp.c index fb47f807..a4cbcfd3 100644 --- a/src/jit/jit-interp.c +++ b/src/jit/jit-interp.c @@ -161,6 +161,10 @@ static jit_scalar_t interp_get_value(jit_interp_t *state, jit_value_t value) }; case JIT_ADDR_ABS: return (jit_scalar_t){ .pointer = (void *)(intptr_t)value.int64 }; + case JIT_ADDR_COVER: + return (jit_scalar_t){ + .pointer = jit_get_cover_ptr(state->func->jit, value) + }; case JIT_VALUE_LABEL: return (jit_scalar_t){ .integer = value.label }; case JIT_VALUE_HANDLE: diff --git a/src/jit/jit-irgen.c b/src/jit/jit-irgen.c index 0aaa98f3..d5209190 100644 --- a/src/jit/jit-irgen.c +++ b/src/jit/jit-irgen.c @@ -23,6 +23,7 @@ #include "lib.h" #include "mask.h" #include "option.h" +#include "rt/cover.h" #include "tree.h" #include "vcode.h" @@ -156,6 +157,14 @@ static jit_value_t jit_addr_from_value(jit_value_t value, int32_t disp) } } +static jit_value_t jit_addr_from_cover_tag(jit_cover_mem_t kind, uint32_t tag) +{ + return (jit_value_t){ + .kind = JIT_ADDR_COVER, + .int64 = kind | ((int64_t)tag << 2) + }; +} + static jit_value_t jit_value_from_label(irgen_label_t *l) { return (jit_value_t){ .kind = JIT_VALUE_LABEL, .label = l->label }; @@ -3178,6 +3187,72 @@ static void irgen_op_driving_value(jit_irgen_t *g, int op) g->map[vcode_get_result(op)] = j_recv(g, 0); } +static void irgen_op_cover_stmt(jit_irgen_t *g, int op) +{ + uint32_t tag = vcode_get_tag(op); + jit_value_t mem = jit_addr_from_cover_tag(JIT_COVER_STMT, tag); + + // XXX: this should be atomic + jit_value_t cur = j_load(g, JIT_SZ_32, mem); + jit_value_t inc = j_add(g, cur, jit_value_from_int64(1)); + j_store(g, JIT_SZ_32, inc, mem); +} + +static void irgen_op_cover_branch(jit_irgen_t *g, int op) +{ + vcode_reg_t arg0 = vcode_get_arg(op, 0); + if (arg0 != g->flags) { + jit_value_t result = irgen_get_arg(g, op, 0); + j_cmp(g, JIT_CC_NE, result, jit_value_from_int64(0)); + } + + uint32_t tag = vcode_get_tag(op); + jit_value_t mem = jit_addr_from_cover_tag(JIT_COVER_BRANCH, tag); + + jit_value_t tval, fval; + if (vcode_get_subkind(op) & COV_FLAG_CHOICE) { + // TODO: just make a seperate cover choice op? + tval = jit_value_from_int64(COV_FLAG_CHOICE); + fval = jit_value_from_int64(0); + } + else { + tval = jit_value_from_int64(COV_FLAG_TRUE); + fval = jit_value_from_int64(COV_FLAG_FALSE); + } + + jit_value_t mask = j_csel(g, tval, fval); + + // XXX: this should be atomic + jit_value_t cur = j_load(g, JIT_SZ_32, mem); + jit_value_t next = j_or(g, cur, mask); + j_store(g, JIT_SZ_32, next, mem); +} + +static void irgen_op_cover_expr(jit_irgen_t *g, int op) +{ + jit_value_t mask = irgen_get_arg(g, op, 0); + + uint32_t tag = vcode_get_tag(op); + jit_value_t mem = jit_addr_from_cover_tag(JIT_COVER_EXPRESSION, tag); + + // XXX: this should be atomic + jit_value_t cur = j_load(g, JIT_SZ_32, mem); + jit_value_t next = j_or(g, cur, mask); + j_store(g, JIT_SZ_32, next, mem); +} + +static void irgen_op_cover_toggle(jit_irgen_t *g, int op) +{ + jit_value_t shared = irgen_get_arg(g, op, 0); + + uint32_t tag = vcode_get_tag(op); + jit_value_t mem = jit_addr_from_cover_tag(JIT_COVER_TOGGLE, tag); + + j_send(g, 0, shared); + j_send(g, 1, mem); + macro_exit(g, JIT_EXIT_COVER_TOGGLE); +} + static void irgen_block(jit_irgen_t *g, vcode_block_t block) { vcode_select_block(block); @@ -3528,6 +3603,18 @@ static void irgen_block(jit_irgen_t *g, vcode_block_t block) case VCODE_OP_DRIVING_VALUE: irgen_op_driving_value(g, i); break; + case VCODE_OP_COVER_STMT: + irgen_op_cover_stmt(g, i); + break; + case VCODE_OP_COVER_BRANCH: + irgen_op_cover_branch(g, i); + break; + case VCODE_OP_COVER_EXPR: + irgen_op_cover_expr(g, i); + break; + case VCODE_OP_COVER_TOGGLE: + irgen_op_cover_toggle(g, i); + break; default: fatal("cannot generate JIT IR for vcode op %s", vcode_op_string(op)); } diff --git a/src/jit/jit-llvm.c b/src/jit/jit-llvm.c index c4230bed..6590ddc1 100644 --- a/src/jit/jit-llvm.c +++ b/src/jit/jit-llvm.c @@ -1013,6 +1013,20 @@ static LLVMValueRef cgen_get_value(llvm_obj_t *obj, cgen_block_t *cgb, return llvm_int32(obj, value.handle); case JIT_ADDR_ABS: return llvm_ptr(obj, (void *)(intptr_t)value.int64); + case JIT_ADDR_COVER: + if (cgb->func->mode == CGEN_AOT) { + LLVMValueRef ptr = cgen_load_from_reloc(obj, cgb->func, + RELOC_COVER, value.int64 & 3); + LLVMValueRef base = LLVMBuildLoad2(obj->builder, + obj->types[LLVM_PTR], ptr, ""); + LLVMValueRef indexes[] = { + llvm_intptr(obj, value.int64 >> 2) + }; + return LLVMBuildGEP2(obj->builder, obj->types[LLVM_INT32], + base, indexes, 1, ""); + } + else + return llvm_ptr(obj, jit_get_cover_ptr(cgb->func->source->jit, value)); case JIT_VALUE_FOREIGN: if (cgb->func->mode == CGEN_AOT) return cgen_load_from_reloc(obj, cgb->func, RELOC_FOREIGN, @@ -2228,13 +2242,27 @@ static void cgen_aot_descr(llvm_obj_t *obj, cgen_func_t *func) tb_printf(tb, "%"PRIi64"\b", ffi_get_spec(args[j].foreign)); tb_istr(tb, ffi_get_sym(args[j].foreign)); - const cgen_reloc_t r1 = { + const cgen_reloc_t r = { .kind = RELOC_FOREIGN, .str = llvm_const_string(obj, tb_get(tb)), .key = (uintptr_t)args[j].foreign, .nth = relocs.count, }; - APUSH(relocs, r1); + APUSH(relocs, r); + } + } + else if (args[j].kind == JIT_ADDR_COVER) { + const jit_cover_mem_t kind = args[j].int64 & 3; + if (cgen_find_reloc(relocs.items, RELOC_COVER, relocs.count, + kind) == NULL) { + const char *map[] = { "stmt", "branch", "toggle", "expr" }; + const cgen_reloc_t r = { + .kind = RELOC_COVER, + .str = llvm_const_string(obj, map[kind]), + .key = kind, + .nth = relocs.count, + }; + APUSH(relocs, r); } } } diff --git a/src/jit/jit-pack.c b/src/jit/jit-pack.c index e2dfae9e..49c4c3a5 100644 --- a/src/jit/jit-pack.c +++ b/src/jit/jit-pack.c @@ -1,5 +1,5 @@ // -// Copyright (C) 2022 Nick Gasson +// Copyright (C) 2022-2023 Nick Gasson // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -203,6 +203,7 @@ static void pack_value(pack_func_t *pf, jit_t *j, jit_value_t value) pack_uint(pf, value.disp); break; case JIT_ADDR_ABS: + case JIT_ADDR_COVER: pack_uint(pf, value.int64); break; case JIT_VALUE_LABEL: @@ -471,6 +472,7 @@ static jit_value_t unpack_value(pack_func_t *pf, jit_t *j) value.disp = unpack_uint(pf); break; case JIT_ADDR_ABS: + case JIT_ADDR_COVER: value.int64 = unpack_uint(pf); break; case JIT_VALUE_LABEL: diff --git a/src/jit/jit-priv.h b/src/jit/jit-priv.h index 6ebdfd7e..35f47972 100644 --- a/src/jit/jit-priv.h +++ b/src/jit/jit-priv.h @@ -1,5 +1,5 @@ // -// Copyright (C) 2022 Nick Gasson +// Copyright (C) 2022-2023 Nick Gasson // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -154,6 +154,7 @@ typedef enum { JIT_EXIT_DRIVING, JIT_EXIT_DRIVING_VALUE, JIT_EXIT_CLAIM_TLAB, + JIT_EXIT_COVER_TOGGLE, } jit_exit_t; typedef uint16_t jit_reg_t; @@ -167,6 +168,7 @@ typedef enum { JIT_ADDR_REG, JIT_ADDR_ABS, JIT_ADDR_CPOOL, + JIT_ADDR_COVER, JIT_VALUE_LABEL, JIT_VALUE_HANDLE, JIT_VALUE_EXIT, @@ -254,6 +256,7 @@ typedef enum { RELOC_FUNC, RELOC_FOREIGN, RELOC_PRIVDATA, + RELOC_COVER, } reloc_kind_t; typedef struct { @@ -340,6 +343,7 @@ int jit_backedge_limit(jit_t *j); void jit_tier_up(jit_func_t *f); jit_thread_local_t *jit_thread_local(void); void jit_fill_irbuf(jit_func_t *f); +int32_t *jit_get_cover_ptr(jit_t *j, jit_value_t addr); jit_cfg_t *jit_get_cfg(jit_func_t *f); void jit_free_cfg(jit_func_t *f); diff --git a/src/jit/jit-x86.c b/src/jit/jit-x86.c index 245ba569..8ed90074 100644 --- a/src/jit/jit-x86.c +++ b/src/jit/jit-x86.c @@ -865,6 +865,9 @@ static void jit_x86_get(code_blob_t *blob, x86_operand_t dst, jit_value_t src) case JIT_ADDR_CPOOL: MOV(dst, IMM((intptr_t)blob->func->cpool + src.int64), __QWORD); break; + case JIT_ADDR_COVER: + MOV(dst, PTR(jit_get_cover_ptr(blob->func->jit, src)), __QWORD); + break; default: fatal_trace("cannot handle value kind %d in jit_x86_get", src.kind); } @@ -883,6 +886,9 @@ static x86_operand_t jit_x86_get_addr(code_blob_t *blob, jit_value_t addr, case JIT_ADDR_ABS: MOV(tmp, IMM(addr.int64), __QWORD); return ADDR(tmp, 0); + case JIT_ADDR_COVER: + MOV(tmp, PTR(jit_get_cover_ptr(blob->func->jit, addr)), __QWORD); + return ADDR(tmp, 0); default: fatal_trace("cannot handle value kind %d in jit_x86_get_addr", addr.kind); } diff --git a/src/jit/jit.h b/src/jit/jit.h index aa276ee8..6ad5eba2 100644 --- a/src/jit/jit.h +++ b/src/jit/jit.h @@ -53,6 +53,13 @@ typedef struct { void (*cleanup)(void *); } jit_plugin_t; +typedef enum { + JIT_COVER_STMT, + JIT_COVER_BRANCH, + JIT_COVER_TOGGLE, + JIT_COVER_EXPRESSION, +} jit_cover_mem_t; + jit_t *jit_new(void); void jit_free(jit_t *j); jit_handle_t jit_compile(jit_t *j, ident_t name); @@ -73,6 +80,10 @@ void jit_add_tier(jit_t *j, int threshold, const jit_plugin_t *plugin); ident_t jit_get_name(jit_t *j, jit_handle_t handle); void jit_register_native_plugin(jit_t *j); +void jit_alloc_cover_mem(jit_t *j, int n_stmts, int n_branches, int n_toggles, + int n_expressions); +int32_t *jit_get_cover_mem(jit_t *j, jit_cover_mem_t kind); + bool jit_try_call(jit_t *j, jit_handle_t handle, jit_scalar_t *result, ...); bool jit_try_call_packed(jit_t *j, jit_handle_t handle, jit_scalar_t context, void *input, size_t insz, void *output, size_t outsz); diff --git a/src/nvc.c b/src/nvc.c index 20f5b9da..9567fdbb 100644 --- a/src/nvc.c +++ b/src/nvc.c @@ -99,6 +99,9 @@ static void bad_option(const char *what, char **argv) { if (optopt == 0) fatal("unrecognised %s option $bold$%s$$", what, argv[optind - 1]); + else if (optarg == NULL) + fatal("%s option $bold$%s$$ requires an argument", + what, argv[optind - 1]); else fatal("unrecognised %s option $bold$-%c$$", what, optopt); } diff --git a/src/rt/model.c b/src/rt/model.c index 2bd7a35f..32bc8c42 100644 --- a/src/rt/model.c +++ b/src/rt/model.c @@ -1819,21 +1819,7 @@ static void reset_coverage(rt_model_t *m) cover_count_tags(m->cover, &n_stmts, &n_branches, &n_toggles, &n_expressions); - int32_t *cover_stmts = ffi_find_symbol(NULL, "cover_stmts"); - if (cover_stmts != NULL) - memset(cover_stmts, '\0', sizeof(int32_t) * n_stmts); - - int32_t *cover_branches = ffi_find_symbol(NULL, "cover_branches"); - if (cover_branches != NULL) - memset(cover_branches, '\0', sizeof(int32_t) * n_branches); - - int32_t *cover_toggles = ffi_find_symbol(NULL, "cover_toggles"); - if (cover_toggles != NULL) - memset(cover_toggles, '\0', sizeof(int32_t) * n_toggles); - - int32_t *cover_expressions = ffi_find_symbol(NULL, "cover_expressions"); - if (cover_expressions != NULL) - memset(cover_expressions, '\0', sizeof(int32_t) * n_expressions); + jit_alloc_cover_mem(m->jit, n_stmts, n_branches, n_toggles, n_expressions); fbuf_close(f, NULL); } @@ -1841,10 +1827,14 @@ static void reset_coverage(rt_model_t *m) static void emit_coverage(rt_model_t *m) { if (m->cover != NULL) { - const int32_t *cover_stmts = ffi_find_symbol(NULL, "cover_stmts"); - const int32_t *cover_branches = ffi_find_symbol(NULL, "cover_branches"); - const int32_t *cover_toggles = ffi_find_symbol(NULL, "cover_toggles"); - const int32_t *cover_expressions = ffi_find_symbol(NULL, "cover_expressions"); + const int32_t *cover_stmts = + jit_get_cover_mem(m->jit, JIT_COVER_STMT); + const int32_t *cover_branches = + jit_get_cover_mem(m->jit, JIT_COVER_BRANCH); + const int32_t *cover_toggles = + jit_get_cover_mem(m->jit, JIT_COVER_TOGGLE); + const int32_t *cover_expressions = + jit_get_cover_mem(m->jit, JIT_COVER_EXPRESSION); fbuf_t *covdb = cover_open_lib_file(m->top, FBUF_OUT, true); cover_dump_tags(m->cover, covdb, COV_DUMP_RUNTIME, cover_stmts, diff --git a/test/run_regr.c b/test/run_regr.c index 981ffff1..b8faf52d 100644 --- a/test/run_regr.c +++ b/test/run_regr.c @@ -671,7 +671,6 @@ static bool run_test(test_t *test) #endif #ifndef HAVE_LLVM skip |= (test->flags & F_SLOW); - skip |= (test->flags & F_COVER); // Only supported with LLVM for now #endif if (skip) { -- 2.39.2