From 2b787d523662b91334da24f1c77a7d803e53fab9 Mon Sep 17 00:00:00 2001 From: Lukas Nykryn Date: Tue, 3 Jan 2017 16:05:42 +0100 Subject: [PATCH] failure-action: generalize failure action to emergency action Cherry-picked from: 87a47f99bc8e576a63581ad2593c62eb10a53814 Resolves: #1353028 --- Makefile.am | 4 +- src/core/dbus-service.c | 6 +-- src/core/dbus-unit.c | 4 +- src/core/{failure-action.c => emergency-action.c} | 65 ++++++++++++----------- src/core/{failure-action.h => emergency-action.h} | 28 +++++----- src/core/job.c | 2 +- src/core/load-fragment-gperf.gperf.m4 | 6 +-- src/core/load-fragment.c | 4 +- src/core/load-fragment.h | 2 +- src/core/manager.h | 2 +- src/core/service.c | 4 +- src/core/service.h | 6 +-- src/core/unit.c | 4 +- src/core/unit.h | 4 +- src/test/test-tables.c | 2 +- 15 files changed, 72 insertions(+), 71 deletions(-) rename src/core/{failure-action.c => emergency-action.c} (63%) rename src/core/{failure-action.h => emergency-action.h} (58%) diff --git a/Makefile.am b/Makefile.am index 3848338a2..b347aed7d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1182,8 +1182,8 @@ libsystemd_core_la_SOURCES = \ src/core/audit-fd.h \ src/core/show-status.c \ src/core/show-status.h \ - src/core/failure-action.c \ - src/core/failure-action.h + src/core/emergency-action.c \ + src/core/emergency-action.h nodist_libsystemd_core_la_SOURCES = \ src/core/load-fragment-gperf.c \ diff --git a/src/core/dbus-service.c b/src/core/dbus-service.c index 6d4713bab..325ed13f5 100644 --- a/src/core/dbus-service.c +++ b/src/core/dbus-service.c @@ -34,7 +34,7 @@ static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_type, service_type, ServiceType static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_result, service_result, ServiceResult); static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_restart, service_restart, ServiceRestart); static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_notify_access, notify_access, NotifyAccess); -static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_failure_action, failure_action, FailureAction); +static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_emergency_action, emergency_action, EmergencyAction); const sd_bus_vtable bus_service_vtable[] = { SD_BUS_VTABLE_START(0), @@ -49,9 +49,9 @@ const sd_bus_vtable bus_service_vtable[] = { BUS_PROPERTY_DUAL_TIMESTAMP("WatchdogTimestamp", offsetof(Service, watchdog_timestamp), 0), SD_BUS_PROPERTY("StartLimitInterval", "t", bus_property_get_usec, offsetof(Service, start_limit.interval), SD_BUS_VTABLE_PROPERTY_CONST), SD_BUS_PROPERTY("StartLimitBurst", "u", bus_property_get_unsigned, offsetof(Service, start_limit.burst), SD_BUS_VTABLE_PROPERTY_CONST), - SD_BUS_PROPERTY("StartLimitAction", "s", property_get_failure_action, offsetof(Service, start_limit_action), SD_BUS_VTABLE_PROPERTY_CONST), + SD_BUS_PROPERTY("StartLimitAction", "s", property_get_emergency_action, offsetof(Service, start_limit_action), SD_BUS_VTABLE_PROPERTY_CONST), SD_BUS_PROPERTY("RebootArgument", "s", NULL, offsetof(Service, reboot_arg), SD_BUS_VTABLE_PROPERTY_CONST), - SD_BUS_PROPERTY("FailureAction", "s", property_get_failure_action, offsetof(Service, failure_action), SD_BUS_VTABLE_PROPERTY_CONST), + SD_BUS_PROPERTY("FailureAction", "s", property_get_emergency_action, offsetof(Service, emergency_action), SD_BUS_VTABLE_PROPERTY_CONST), SD_BUS_PROPERTY("PermissionsStartOnly", "b", bus_property_get_bool, offsetof(Service, permissions_start_only), SD_BUS_VTABLE_PROPERTY_CONST), SD_BUS_PROPERTY("RootDirectoryStartOnly", "b", bus_property_get_bool, offsetof(Service, root_directory_start_only), SD_BUS_VTABLE_PROPERTY_CONST), SD_BUS_PROPERTY("RemainAfterExit", "b", bus_property_get_bool, offsetof(Service, remain_after_exit), SD_BUS_VTABLE_PROPERTY_CONST), diff --git a/src/core/dbus-unit.c b/src/core/dbus-unit.c index c3654db9e..89b00e94c 100644 --- a/src/core/dbus-unit.c +++ b/src/core/dbus-unit.c @@ -33,7 +33,7 @@ static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_load_state, unit_load_state, UnitLoadState); static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_job_mode, job_mode, JobMode); -static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_failure_action, failure_action, FailureAction); +static BUS_DEFINE_PROPERTY_GET_ENUM(property_get_emergency_action, emergency_action, EmergencyAction); static int property_get_names( sd_bus *bus, @@ -595,7 +595,7 @@ const sd_bus_vtable bus_unit_vtable[] = { SD_BUS_PROPERTY("IgnoreOnSnapshot", "b", bus_property_get_bool, offsetof(Unit, ignore_on_snapshot), SD_BUS_VTABLE_PROPERTY_CONST), SD_BUS_PROPERTY("NeedDaemonReload", "b", property_get_need_daemon_reload, 0, SD_BUS_VTABLE_PROPERTY_CONST), SD_BUS_PROPERTY("JobTimeoutUSec", "t", bus_property_get_usec, offsetof(Unit, job_timeout), SD_BUS_VTABLE_PROPERTY_CONST), - SD_BUS_PROPERTY("JobTimeoutAction", "s", property_get_failure_action, offsetof(Unit, job_timeout_action), SD_BUS_VTABLE_PROPERTY_CONST), + SD_BUS_PROPERTY("JobTimeoutAction", "s", property_get_emergency_action, offsetof(Unit, job_timeout_action), SD_BUS_VTABLE_PROPERTY_CONST), SD_BUS_PROPERTY("JobTimeoutRebootArgument", "s", NULL, offsetof(Unit, job_timeout_reboot_arg), SD_BUS_VTABLE_PROPERTY_CONST), SD_BUS_PROPERTY("ConditionResult", "b", bus_property_get_bool, offsetof(Unit, condition_result), SD_BUS_VTABLE_PROPERTY_EMITS_CHANGE), SD_BUS_PROPERTY("AssertResult", "b", bus_property_get_bool, offsetof(Unit, assert_result), SD_BUS_VTABLE_PROPERTY_EMITS_CHANGE), diff --git a/src/core/failure-action.c b/src/core/emergency-action.c similarity index 63% rename from src/core/failure-action.c rename to src/core/emergency-action.c index ce522a4e4..f07b1257a 100644 --- a/src/core/failure-action.c +++ b/src/core/emergency-action.c @@ -27,44 +27,45 @@ #include "bus-util.h" #include "bus-error.h" #include "special.h" -#include "failure-action.h" +#include "emergency-action.h" -static void log_and_status(Manager *m, const char *message) { - log_warning("%s", message); +static void log_and_status(Manager *m, const char *message, const char *reason) { + log_warning("%s: %s", message, reason); manager_status_printf(m, STATUS_TYPE_EMERGENCY, ANSI_HIGHLIGHT_RED_ON " !! " ANSI_HIGHLIGHT_OFF, - "%s", message); + "%s: %s", message, reason); } -int failure_action( +int emergency_action( Manager *m, - FailureAction action, - const char *reboot_arg) { + EmergencyAction action, + const char *reboot_arg, + const char *reason) { int r; assert(m); assert(action >= 0); - assert(action < _FAILURE_ACTION_MAX); + assert(action < _EMERGENCY_ACTION_MAX); - if (action == FAILURE_ACTION_NONE) + if (action == EMERGENCY_ACTION_NONE) return -ECANCELED; if (m->running_as == SYSTEMD_USER) { /* Downgrade all options to simply exiting if we run * in user mode */ - log_warning("Exiting as result of failure."); + log_warning("Exiting: %s", reason); m->exit_code = MANAGER_EXIT; return -ECANCELED; } switch (action) { - case FAILURE_ACTION_REBOOT: { + case EMERGENCY_ACTION_REBOOT: { _cleanup_bus_error_free_ sd_bus_error error = SD_BUS_ERROR_NULL; - log_and_status(m, "Rebooting as result of failure."); + log_and_status(m, "Rebooting", reason); update_reboot_param_file(reboot_arg); r = manager_add_job_by_name(m, JOB_START, SPECIAL_REBOOT_TARGET, JOB_REPLACE, true, &error, NULL); @@ -74,15 +75,15 @@ int failure_action( break; } - case FAILURE_ACTION_REBOOT_FORCE: - log_and_status(m, "Forcibly rebooting as result of failure."); + case EMERGENCY_ACTION_REBOOT_FORCE: + log_and_status(m, "Forcibly rebooting", reason); update_reboot_param_file(reboot_arg); m->exit_code = MANAGER_REBOOT; break; - case FAILURE_ACTION_REBOOT_IMMEDIATE: - log_and_status(m, "Rebooting immediately as result of failure."); + case EMERGENCY_ACTION_REBOOT_IMMEDIATE: + log_and_status(m, "Rebooting immediately", reason); sync(); @@ -95,10 +96,10 @@ int failure_action( reboot(RB_AUTOBOOT); break; - case FAILURE_ACTION_POWEROFF: { + case EMERGENCY_ACTION_POWEROFF: { _cleanup_bus_error_free_ sd_bus_error error = SD_BUS_ERROR_NULL; - log_and_status(m, "Powering off as result of failure."); + log_and_status(m, "Powering off", reason); r = manager_add_job_by_name(m, JOB_START, SPECIAL_POWEROFF_TARGET, JOB_REPLACE, true, &error, NULL); if (r < 0) @@ -107,13 +108,13 @@ int failure_action( break; } - case FAILURE_ACTION_POWEROFF_FORCE: - log_and_status(m, "Forcibly powering off as result of failure."); + case EMERGENCY_ACTION_POWEROFF_FORCE: + log_and_status(m, "Forcibly powering off", reason); m->exit_code = MANAGER_POWEROFF; break; - case FAILURE_ACTION_POWEROFF_IMMEDIATE: - log_and_status(m, "Powering off immediately as result of failure."); + case EMERGENCY_ACTION_POWEROFF_IMMEDIATE: + log_and_status(m, "Powering off immediately", reason); sync(); @@ -122,19 +123,19 @@ int failure_action( break; default: - assert_not_reached("Unknown failure action"); + assert_not_reached("Unknown emergency action"); } return -ECANCELED; } -static const char* const failure_action_table[_FAILURE_ACTION_MAX] = { - [FAILURE_ACTION_NONE] = "none", - [FAILURE_ACTION_REBOOT] = "reboot", - [FAILURE_ACTION_REBOOT_FORCE] = "reboot-force", - [FAILURE_ACTION_REBOOT_IMMEDIATE] = "reboot-immediate", - [FAILURE_ACTION_POWEROFF] = "poweroff", - [FAILURE_ACTION_POWEROFF_FORCE] = "poweroff-force", - [FAILURE_ACTION_POWEROFF_IMMEDIATE] = "poweroff-immediate" +static const char* const emergency_action_table[_EMERGENCY_ACTION_MAX] = { + [EMERGENCY_ACTION_NONE] = "none", + [EMERGENCY_ACTION_REBOOT] = "reboot", + [EMERGENCY_ACTION_REBOOT_FORCE] = "reboot-force", + [EMERGENCY_ACTION_REBOOT_IMMEDIATE] = "reboot-immediate", + [EMERGENCY_ACTION_POWEROFF] = "poweroff", + [EMERGENCY_ACTION_POWEROFF_FORCE] = "poweroff-force", + [EMERGENCY_ACTION_POWEROFF_IMMEDIATE] = "poweroff-immediate" }; -DEFINE_STRING_TABLE_LOOKUP(failure_action, FailureAction); +DEFINE_STRING_TABLE_LOOKUP(emergency_action, EmergencyAction); diff --git a/src/core/failure-action.h b/src/core/emergency-action.h similarity index 58% rename from src/core/failure-action.h rename to src/core/emergency-action.h index 1af4dd987..d3ac0f31c 100644 --- a/src/core/failure-action.h +++ b/src/core/emergency-action.h @@ -22,22 +22,22 @@ along with systemd; If not, see . ***/ -typedef enum FailureAction { - FAILURE_ACTION_NONE, - FAILURE_ACTION_REBOOT, - FAILURE_ACTION_REBOOT_FORCE, - FAILURE_ACTION_REBOOT_IMMEDIATE, - FAILURE_ACTION_POWEROFF, - FAILURE_ACTION_POWEROFF_FORCE, - FAILURE_ACTION_POWEROFF_IMMEDIATE, - _FAILURE_ACTION_MAX, - _FAILURE_ACTION_INVALID = -1 -} FailureAction; +typedef enum EmergencyAction { + EMERGENCY_ACTION_NONE, + EMERGENCY_ACTION_REBOOT, + EMERGENCY_ACTION_REBOOT_FORCE, + EMERGENCY_ACTION_REBOOT_IMMEDIATE, + EMERGENCY_ACTION_POWEROFF, + EMERGENCY_ACTION_POWEROFF_FORCE, + EMERGENCY_ACTION_POWEROFF_IMMEDIATE, + _EMERGENCY_ACTION_MAX, + _EMERGENCY_ACTION_INVALID = -1 +} EmergencyAction; #include "macro.h" #include "manager.h" -int failure_action(Manager *m, FailureAction action, const char *reboot_arg); +int emergency_action(Manager *m, EmergencyAction action, const char *reboot_arg, const char *reason); -const char* failure_action_to_string(FailureAction i) _const_; -FailureAction failure_action_from_string(const char *s) _pure_; +const char* emergency_action_to_string(EmergencyAction i) _const_; +EmergencyAction emergency_action_from_string(const char *s) _pure_; diff --git a/src/core/job.c b/src/core/job.c index c2876dec1..703286496 100644 --- a/src/core/job.c +++ b/src/core/job.c @@ -916,7 +916,7 @@ static int job_dispatch_timer(sd_event_source *s, uint64_t monotonic, void *user u = j->unit; job_finish_and_invalidate(j, JOB_TIMEOUT, true, false); - failure_action(u->manager, u->job_timeout_action, u->job_timeout_reboot_arg); + emergency_action(u->manager, u->job_timeout_action, u->job_timeout_reboot_arg, "job timed out"); return 0; } diff --git a/src/core/load-fragment-gperf.gperf.m4 b/src/core/load-fragment-gperf.gperf.m4 index ce1397c7e..45d1ead45 100644 --- a/src/core/load-fragment-gperf.gperf.m4 +++ b/src/core/load-fragment-gperf.gperf.m4 @@ -156,7 +156,7 @@ Unit.OnFailureIsolate, config_parse_job_mode_isolate, 0, Unit.IgnoreOnIsolate, config_parse_bool, 0, offsetof(Unit, ignore_on_isolate) Unit.IgnoreOnSnapshot, config_parse_bool, 0, offsetof(Unit, ignore_on_snapshot) Unit.JobTimeoutSec, config_parse_sec, 0, offsetof(Unit, job_timeout) -Unit.JobTimeoutAction, config_parse_failure_action, 0, offsetof(Unit, job_timeout_action) +Unit.JobTimeoutAction, config_parse_emergency_action, 0, offsetof(Unit, job_timeout_action) Unit.JobTimeoutRebootArgument, config_parse_string, 0, offsetof(Unit, job_timeout_reboot_arg) Unit.ConditionPathExists, config_parse_unit_condition_path, CONDITION_PATH_EXISTS, offsetof(Unit, conditions) Unit.ConditionPathExistsGlob, config_parse_unit_condition_path, CONDITION_PATH_EXISTS_GLOB, offsetof(Unit, conditions) @@ -211,9 +211,9 @@ Service.TimeoutStopSec, config_parse_service_timeout, 0, Service.WatchdogSec, config_parse_sec, 0, offsetof(Service, watchdog_usec) Service.StartLimitInterval, config_parse_sec, 0, offsetof(Service, start_limit.interval) Service.StartLimitBurst, config_parse_unsigned, 0, offsetof(Service, start_limit.burst) -Service.StartLimitAction, config_parse_failure_action, 0, offsetof(Service, start_limit_action) +Service.StartLimitAction, config_parse_emergency_action, 0, offsetof(Service, start_limit_action) Service.RebootArgument, config_parse_string, 0, offsetof(Service, reboot_arg) -Service.FailureAction, config_parse_failure_action, 0, offsetof(Service, failure_action) +Service.FailureAction, config_parse_emergency_action, 0, offsetof(Service, emergency_action) Service.Type, config_parse_service_type, 0, offsetof(Service, type) Service.Restart, config_parse_service_restart, 0, offsetof(Service, restart) Service.PermissionsStartOnly, config_parse_bool, 0, offsetof(Service, permissions_start_only) diff --git a/src/core/load-fragment.c b/src/core/load-fragment.c index 83b6e7efc..4fecb8314 100644 --- a/src/core/load-fragment.c +++ b/src/core/load-fragment.c @@ -2382,7 +2382,7 @@ int config_parse_unit_condition_null( } DEFINE_CONFIG_PARSE_ENUM(config_parse_notify_access, notify_access, NotifyAccess, "Failed to parse notify access specifier"); -DEFINE_CONFIG_PARSE_ENUM(config_parse_failure_action, failure_action, FailureAction, "Failed to parse failure action specifier"); +DEFINE_CONFIG_PARSE_ENUM(config_parse_emergency_action, emergency_action, EmergencyAction, "Failed to parse failure action specifier"); int config_parse_unit_requires_mounts_for( const char *unit, @@ -3931,7 +3931,7 @@ void unit_dump_config_items(FILE *f) { { config_parse_unit_slice, "SLICE" }, { config_parse_documentation, "URL" }, { config_parse_service_timeout, "SECONDS" }, - { config_parse_failure_action, "ACTION" }, + { config_parse_emergency_action, "ACTION" }, { config_parse_set_status, "STATUS" }, { config_parse_service_sockets, "SOCKETS" }, { config_parse_environ, "ENVIRON" }, diff --git a/src/core/load-fragment.h b/src/core/load-fragment.h index 359794d0a..611479612 100644 --- a/src/core/load-fragment.h +++ b/src/core/load-fragment.h @@ -78,7 +78,7 @@ int config_parse_unit_condition_string(const char *unit, const char *filename, u int config_parse_unit_condition_null(const char *unit, const char *filename, unsigned line, const char *section, unsigned section_line, const char *lvalue, int ltype, const char *rvalue, void *data, void *userdata); int config_parse_kill_mode(const char *unit, const char *filename, unsigned line, const char *section, unsigned section_line, const char *lvalue, int ltype, const char *rvalue, void *data, void *userdata); int config_parse_notify_access(const char *unit, const char *filename, unsigned line, const char *section, unsigned section_line, const char *lvalue, int ltype, const char *rvalue, void *data, void *userdata); -int config_parse_failure_action(const char *unit, const char *filename, unsigned line, const char *section, unsigned section_line, const char *lvalue, int ltype, const char *rvalue, void *data, void *userdata); +int config_parse_emergency_action(const char *unit, const char *filename, unsigned line, const char *section, unsigned section_line, const char *lvalue, int ltype, const char *rvalue, void *data, void *userdata); int config_parse_unit_requires_mounts_for(const char *unit, const char *filename, unsigned line, const char *section, unsigned section_line, const char *lvalue, int ltype, const char *rvalue, void *data, void *userdata); int config_parse_syscall_filter(const char *unit, const char *filename, unsigned line, const char *section, unsigned section_line, const char *lvalue, int ltype, const char *rvalue, void *data, void *userdata); int config_parse_syscall_archs(const char *unit, const char *filename, unsigned line, const char *section, unsigned section_line, const char *lvalue, int ltype, const char *rvalue, void *data, void *userdata); diff --git a/src/core/manager.h b/src/core/manager.h index 42be1fc43..59913f489 100644 --- a/src/core/manager.h +++ b/src/core/manager.h @@ -85,7 +85,7 @@ typedef enum StatusType { #include "unit-name.h" #include "exit-status.h" #include "show-status.h" -#include "failure-action.h" +#include "emergency-action.h" struct Manager { /* Note that the set of units we know of is allowed to be diff --git a/src/core/service.c b/src/core/service.c index babd3c52a..6e7baa76c 100644 --- a/src/core/service.c +++ b/src/core/service.c @@ -1280,7 +1280,7 @@ static void service_enter_dead(Service *s, ServiceResult f, bool allow_restart) if (s->result != SERVICE_SUCCESS) { log_unit_warning(UNIT(s)->id, "%s failed.", UNIT(s)->id); - failure_action(UNIT(s)->manager, s->failure_action, s->reboot_arg); + emergency_action(UNIT(s)->manager, s->emergency_action, s->reboot_arg, "service failed"); } if (allow_restart && @@ -1821,7 +1821,7 @@ static int service_start_limit_test(Service *s) { log_unit_warning(UNIT(s)->id, "start request repeated too quickly for %s", UNIT(s)->id); - return failure_action(UNIT(s)->manager, s->start_limit_action, s->reboot_arg); + return emergency_action(UNIT(s)->manager, s->start_limit_action, s->reboot_arg, "service failed"); } static int service_start(Unit *u) { diff --git a/src/core/service.h b/src/core/service.h index dfeee6a68..1f937dfe5 100644 --- a/src/core/service.h +++ b/src/core/service.h @@ -29,7 +29,7 @@ typedef struct ServiceFDStore ServiceFDStore; #include "ratelimit.h" #include "kill.h" #include "exit-status.h" -#include "failure-action.h" +#include "emergency-action.h" typedef enum ServiceState { SERVICE_DEAD, @@ -197,8 +197,8 @@ struct Service { int status_errno; RateLimit start_limit; - FailureAction start_limit_action; - FailureAction failure_action; + EmergencyAction start_limit_action; + EmergencyAction emergency_action; char *reboot_arg; UnitRef accept_socket; diff --git a/src/core/unit.c b/src/core/unit.c index a7d6d2f45..4eb0d78f4 100644 --- a/src/core/unit.c +++ b/src/core/unit.c @@ -937,8 +937,8 @@ void unit_dump(Unit *u, FILE *f, const char *prefix) { if (u->job_timeout > 0) fprintf(f, "%s\tJob Timeout: %s\n", prefix, format_timespan(timespan, sizeof(timespan), u->job_timeout, 0)); - if (u->job_timeout_action != FAILURE_ACTION_NONE) - fprintf(f, "%s\tJob Timeout Action: %s\n", prefix, failure_action_to_string(u->job_timeout_action)); + if (u->job_timeout_action != EMERGENCY_ACTION_NONE) + fprintf(f, "%s\tJob Timeout Action: %s\n", prefix, emergency_action_to_string(u->job_timeout_action)); if (u->job_timeout_reboot_arg) fprintf(f, "%s\tJob Timeout Reboot Argument: %s\n", prefix, u->job_timeout_reboot_arg); diff --git a/src/core/unit.h b/src/core/unit.h index 35287a5b7..85f52df18 100644 --- a/src/core/unit.h +++ b/src/core/unit.h @@ -41,7 +41,7 @@ typedef struct UnitStatusMessageFormats UnitStatusMessageFormats; #include "condition.h" #include "install.h" #include "unit-name.h" -#include "failure-action.h" +#include "emergency-action.h" enum UnitActiveState { UNIT_ACTIVE, @@ -121,7 +121,7 @@ struct Unit { /* Job timeout and action to take */ usec_t job_timeout; - FailureAction job_timeout_action; + EmergencyAction job_timeout_action; char *job_timeout_reboot_arg; /* References to this */ diff --git a/src/test/test-tables.c b/src/test/test-tables.c index bda224bec..e4097903c 100644 --- a/src/test/test-tables.c +++ b/src/test/test-tables.c @@ -67,7 +67,7 @@ int main(int argc, char **argv) { test_table(device_state, DEVICE_STATE); test_table(exec_input, EXEC_INPUT); test_table(exec_output, EXEC_OUTPUT); - test_table(failure_action, FAILURE_ACTION); + test_table(emergency_action, EMERGENCY_ACTION); test_table(job_mode, JOB_MODE); test_table(job_result, JOB_RESULT); test_table(job_state, JOB_STATE);