/* * Copyright (C) 1994-2021 Altair Engineering, Inc. * For more information, contact Altair at www.altair.com. * * This file is part of both the OpenPBS software ("OpenPBS") * and the PBS Professional ("PBS Pro") software. * * Open Source License Information: * * OpenPBS is free software. You can redistribute it and/or modify it under * the terms of the GNU Affero General Public License as published by the * Free Software Foundation, either version 3 of the License, or (at your * option) any later version. * * OpenPBS is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public * License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . * * Commercial License Information: * * PBS Pro is commercially licensed software that shares a common core with * the OpenPBS software. For a copy of the commercial license terms and * conditions, go to: (http://www.pbspro.com/agreement.html) or contact the * Altair Legal Department. * * Altair's dual-license business model allows companies, individuals, and * organizations to create proprietary derivative works of OpenPBS and * distribute them - whether embedded or bundled with other software - * under a commercial license agreement. * * Use of Altair's trademarks, including but not limited to "PBS™", * "OpenPBS®", "PBS Professional®", and "PBS Pro™" and Altair's logos is * subject to Altair's trademark licensing policies. */ /** * * @brief * contains functions to initialize several pbs data structures. * */ #include /* the master config generated by configure */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "libpbs.h" #include "pbs_ifl.h" #include "net_connect.h" #include "log.h" #include "list_link.h" #include "attribute.h" #include "server_limits.h" #include "server.h" #include "credential.h" #include "ticket.h" #include "batch_request.h" #include "work_task.h" #include "resv_node.h" #include "job.h" #include "queue.h" #include "reservation.h" #include "pbs_db.h" #include "pbs_nodes.h" #include "tracking.h" #include "provision.h" #include "pbs_idx.h" #include "svrfunc.h" #include "acct.h" #include "pbs_version.h" #include "tpp.h" #include "pbs_license.h" #include "resource.h" #include "pbs_python.h" #include "hook.h" #include "hook_func.h" #include "pbs_share.h" #include "liblicense.h" #ifndef SIGKILL /* there is some weid stuff in gcc include files signal.h & sys/params.h */ #include #endif /* global Data Items */ extern char *msg_startup3; extern char *msg_daemonname; extern char *msg_init_abt; extern char *msg_init_queued; extern char *msg_init_substate; extern char *msg_err_noqueue; extern char *msg_err_noqueue1; extern char *msg_init_resvNOq; extern char *msg_init_recovque; extern char *msg_init_recovresv; extern char *msg_init_expctq; extern char *msg_init_nojobs; extern char *msg_init_exptjobs; extern char *msg_init_unkstate; extern char *msg_init_baddb; extern char *msg_init_chdir; extern char *msg_unkresc; extern char *msg_corelimit; extern char *acct_file; extern int ext_license_server; extern char *log_file; extern char *path_acct; extern char *path_usedlicenses; extern char path_log[]; extern char *path_priv; extern char *path_jobs; extern char *path_users; extern char *path_spool; extern char *path_track; extern char *path_prov_track; extern long new_log_event_mask; extern char server_name[]; extern pbs_list_head svr_newjobs; extern pbs_list_head svr_allresvs; extern time_t time_now; extern struct server server; extern struct attribute attr_jobscript_max_size; extern char *path_hooks; extern char *path_hooks_workdir; extern pbs_list_head prov_allvnodes; extern int max_concurrent_prov; extern void *svr_db_conn; extern pbs_list_head svr_allhooks; /* External Functions Called */ extern void on_job_exit(struct work_task *); extern void on_job_rerun(struct work_task *); extern int resize_prov_table(int newsize); extern void offline_all_provisioning_vnodes(void); extern void stop_db(); extern job *job_recov_db_spl(pbs_db_job_info_t *dbjob, job *pjob); extern pbs_sched *sched_alloc(char *sched_name); extern job *recov_job_cb(pbs_db_obj_info_t *, int *); extern resc_resv *recov_resv_cb(pbs_db_obj_info_t *, int *); extern pbs_queue *recov_queue_cb(pbs_db_obj_info_t *, int *); extern pbs_sched *recov_sched_cb(pbs_db_obj_info_t *, int *); extern void revert_alter_reservation(resc_resv *presv); extern void log_licenses(pbs_licenses_high_use *pu); /* Private functions in this file */ static void catch_child(int); static void init_abt_job(job *); static void change_logs(int); int chk_save_file(char *filename); static void need_y_response(int, char *); static int pbsd_init_reque(job *job, int change_state); static void resume_net_move(struct work_task *); static void stop_me(int); static int Rmv_if_resv_not_possible(job *); static int attach_queue_to_reservation(resc_resv *); static void call_log_license(struct work_task *); /* private data */ #define CHANGE_STATE 1 #define KEEP_STATE 0 static char badlicense[] = "One or more PBS license keys are invalid, jobs may not run"; char *pbs_licensing_location = NULL; /** * @brief * Initializes the server attribute array with default values which are * necessary for recovery and action routines to work properly. * * @return void */ void init_server_attrs() { resource_def *prdef = NULL; resource *presc = NULL; int i = 0; for (i = 0; i < SVR_ATR_LAST; i++) clear_attr(get_sattr(i), &svr_attr_def[i]); set_sattr_str_slim(SVR_ATR_scheduler_iteration, TOSTR(PBS_SCHEDULE_CYCLE), NULL); server.newobj = 1; set_sattr_l_slim(SVR_ATR_State, SV_STATE_INIT, SET); set_sattr_l_slim(SVR_ATR_ResvEnable, 1, SET); (get_sattr(SVR_ATR_ResvEnable))->at_flags |= ATR_VFLAG_DEFLT; set_sattr_str_slim(SVR_ATR_SvrHost, server_host, NULL); (get_sattr(SVR_ATR_SvrHost))->at_flags |= ATR_VFLAG_DEFLT; set_sattr_l_slim(SVR_ATR_NodeFailReq, PBS_NODE_FAIL_REQUEUE_DEFAULT, SET); (get_sattr(SVR_ATR_NodeFailReq))->at_flags |= ATR_VFLAG_DEFLT; set_sattr_l_slim(SVR_ATR_maxarraysize, PBS_MAX_ARRAY_JOB_DFL, SET); (get_sattr(SVR_ATR_maxarraysize))->at_flags |= ATR_VFLAG_DEFLT; set_sattr_l_slim(SVR_ATR_license_min, PBS_MIN_LICENSING_LICENSES, SET); (get_sattr(SVR_ATR_license_min))->at_flags |= ATR_VFLAG_DEFLT; licensing_control.licenses_min = PBS_MIN_LICENSING_LICENSES; set_sattr_l_slim(SVR_ATR_license_max, PBS_MAX_LICENSING_LICENSES, SET); (get_sattr(SVR_ATR_license_max))->at_flags |= ATR_VFLAG_DEFLT; licensing_control.licenses_max = PBS_MAX_LICENSING_LICENSES; set_sattr_l_slim(SVR_ATR_license_linger, PBS_LIC_LINGER_TIME, SET); (get_sattr(SVR_ATR_license_linger))->at_flags |= ATR_VFLAG_DEFLT; licensing_control.licenses_linger_time = PBS_LIC_LINGER_TIME; set_sattr_l_slim(SVR_ATR_EligibleTimeEnable, 0, SET); (get_sattr(SVR_ATR_EligibleTimeEnable))->at_flags |= ATR_VFLAG_DEFLT; set_sattr_l_slim(SVR_ATR_max_concurrent_prov, PBS_MAX_CONCURRENT_PROV, SET); (get_sattr(SVR_ATR_max_concurrent_prov))->at_flags |= ATR_VFLAG_DEFLT; set_sattr_str_slim(SVR_ATR_max_job_sequence_id, TOSTR(SVR_MAX_JOB_SEQ_NUM_DEFAULT), NULL); (get_sattr(SVR_ATR_max_job_sequence_id))->at_flags |= ATR_VFLAG_DEFLT; set_attr_generic(&attr_jobscript_max_size, &svr_attr_def[SVR_ATR_jobscript_max_size], DFLT_JOBSCRIPT_MAX_SIZE, NULL, INTERNAL); attr_jobscript_max_size.at_type = ATR_TYPE_SIZE; /* needed by get_bytes_from_attr */ set_sattr_l_slim(SVR_ATR_has_runjob_hook, 0, SET); set_sattr_l_slim(SVR_ATR_log_events, SVR_LOG_DFLT, SET); *log_event_mask = get_sattr_long(SVR_ATR_log_events); set_sattr_str_slim(SVR_ATR_mailer, SENDMAIL_CMD, NULL); set_sattr_str_slim(SVR_ATR_mailfrom, PBS_DEFAULT_MAIL, NULL); set_sattr_l_slim(SVR_ATR_query_others, 1, SET); set_sattr_l_slim(SVR_ATR_scheduling, 1, SET); prdef = &svr_resc_def[RESC_NCPUS]; if (prdef) { attribute *pattr = get_sattr(SVR_ATR_DefaultChunk); presc = add_resource_entry(pattr, prdef); if (presc) { presc->rs_value.at_val.at_long = 1; presc->rs_value.at_flags = ATR_VFLAG_DEFLT | ATR_SET_MOD_MCACHE; pattr->at_flags = ATR_VFLAG_DEFLT | ATR_SET_MOD_MCACHE; (void) deflt_chunk_action(pattr, (void *) &server, ATR_ACTION_NEW); } pattr = get_sattr(SVR_ATR_resource_deflt); presc = add_resource_entry(pattr, prdef); if (presc) { presc->rs_value.at_val.at_long = 1; presc->rs_value.at_flags = ATR_VFLAG_DEFLT | ATR_SET_MOD_MCACHE; pattr->at_flags = ATR_VFLAG_DEFLT | ATR_SET_MOD_MCACHE; } } } /** * @brief * This file contains the functions to initialize the PBS Batch Server. * The code is called once when the server is brought up. * * @param[in] type - The type of initialization * RECOV_CREATE - reinitializes all serverdb data * * @return Error code * @retval 0 - Success * @retval Non-Zero - Failure * */ int pbsd_init(int type) { int a_opt = -1; int baselen; struct dirent *pdirent; DIR *dir; int fd; int i = 0; char zone_dir[MAXPATHLEN]; char *hook_suffix = HOOK_FILE_SUFFIX; int hook_suf_len = strlen(hook_suffix); hook *phook, *phook_current; char *psuffix; int rc; struct stat statbuf; char hook_msg[HOOK_MSG_SIZE]; char *conn_db_err = NULL; struct sigaction act; struct sigaction oact; struct tm *ptm; pbs_db_job_info_t dbjob = {{0}}; pbs_db_resv_info_t dbresv = {{0}}; pbs_db_que_info_t dbque = {{0}}; pbs_db_sched_info_t dbsched = {{0}}; pbs_db_obj_info_t obj = {0}; void *conn = (void *) svr_db_conn; char *buf = NULL; int buf_len = 0; #ifdef RLIMIT_CORE int char_in_cname = 0; #endif /* RLIMIT_CORE */ if ((job_attr_idx = cr_attrdef_idx(job_attr_def, JOB_ATR_LAST)) == NULL) { log_err(errno, __func__, "Failed creating job attribute search index"); return (-1); } if ((node_attr_idx = cr_attrdef_idx(node_attr_def, ND_ATR_LAST)) == NULL) { log_err(errno, __func__, "Failed creating node attribute search index"); return (-1); } if ((que_attr_idx = cr_attrdef_idx(que_attr_def, QA_ATR_LAST)) == NULL) { log_err(errno, __func__, "Failed creating queue attribute search index"); return (-1); } if ((svr_attr_idx = cr_attrdef_idx(svr_attr_def, SVR_ATR_LAST)) == NULL) { log_err(errno, __func__, "Failed creating server attribute search index"); return (-1); } if ((sched_attr_idx = cr_attrdef_idx(sched_attr_def, SCHED_ATR_LAST)) == NULL) { log_err(errno, __func__, "Failed creating sched attribute search index"); return (-1); } if ((resv_attr_idx = cr_attrdef_idx(resv_attr_def, RESV_ATR_LAST)) == NULL) { log_err(errno, __func__, "Failed creating resv attribute search index"); return (-1); } if (cr_rescdef_idx(svr_resc_def, svr_resc_size) != 0) { log_err(errno, __func__, "Failed creating resc definition search index"); return (-1); } /* initialize the pointers in the resource_def array */ for (i = 0; i < (svr_resc_size - 1); ++i) svr_resc_def[i].rs_next = &svr_resc_def[i + 1]; /* last entry is left with null pointer */ /* The following is code to reduce security risks */ log_supported_auth_methods(pbs_conf.supported_auth_methods); i = getgid(); (void) setgroups(1, (gid_t *) &i); /* secure suppl. groups */ #ifdef RLIMIT_CORE if (pbs_conf.pbs_core_limit) { char *pc = pbs_conf.pbs_core_limit; while (*pc != '\0') { if (!isdigit(*pc)) { /* there is a character in core limit */ char_in_cname = 1; break; } pc++; } } #endif /* RLIMIT_CORE */ { struct rlimit rlimit; rlimit.rlim_cur = RLIM_INFINITY; rlimit.rlim_max = RLIM_INFINITY; (void) setrlimit(RLIMIT_CPU, &rlimit); (void) setrlimit(RLIMIT_FSIZE, &rlimit); (void) setrlimit(RLIMIT_DATA, &rlimit); (void) setrlimit(RLIMIT_STACK, &rlimit); #ifdef RLIMIT_RSS (void) setrlimit(RLIMIT_RSS, &rlimit); #endif /* RLIMIT_RSS */ #ifdef RLIMIT_VMEM (void) setrlimit(RLIMIT_VMEM, &rlimit); #endif /* RLIMIT_VMEM */ #ifdef RLIMIT_CORE if (pbs_conf.pbs_core_limit) { struct rlimit corelimit; corelimit.rlim_max = RLIM_INFINITY; if (strcmp("unlimited", pbs_conf.pbs_core_limit) == 0) corelimit.rlim_cur = RLIM_INFINITY; else if (char_in_cname == 1) { log_record(PBSEVENT_ERROR, PBS_EVENTCLASS_SERVER, LOG_WARNING, __func__, msg_corelimit); corelimit.rlim_cur = RLIM_INFINITY; } else corelimit.rlim_cur = (rlim_t) atol(pbs_conf.pbs_core_limit); (void) setrlimit(RLIMIT_CORE, &corelimit); } #endif /* RLIMIT_CORE */ } /* 1. set up to catch or ignore various signals */ sigemptyset(&act.sa_mask); act.sa_flags = 0; act.sa_handler = change_logs; if (sigaction(SIGHUP, &act, &oact) != 0) { log_err(errno, __func__, "sigaction for HUP"); return (2); } act.sa_handler = stop_me; if (sigaction(SIGINT, &act, &oact) != 0) { log_err(errno, __func__, "sigaction for INT"); return (2); } if (sigaction(SIGTERM, &act, &oact) != 0) { log_err(errno, __func__, "sigactin for TERM"); return (2); } #ifdef NDEBUG if (sigaction(SIGQUIT, &act, &oact) != 0) { log_err(errno, __func__, "sigactin for QUIT"); return (2); } #endif /* NDEBUG */ #ifdef SIGSHUTDN if (sigaction(SIGSHUTDN, &act, &oact) != 0) { log_err(errno, __func__, "sigactin for SHUTDN"); return (2); } #endif /* SIGSHUTDN */ act.sa_handler = catch_child; if (sigaction(SIGCHLD, &act, &oact) != 0) { log_err(errno, __func__, "sigaction for CHLD"); return (2); } act.sa_handler = SIG_IGN; if (sigaction(SIGPIPE, &act, &oact) != 0) { log_err(errno, __func__, "sigaction for PIPE"); return (2); } if (sigaction(SIGUSR1, &act, &oact) != 0) { log_err(errno, __func__, "sigaction for USR1"); return (2); } if (sigaction(SIGUSR2, &act, &oact) != 0) { log_err(errno, __func__, "sigaction for USR2"); return (2); } /* 2. check security and set up various global variables we need */ #if !defined(DEBUG) && !defined(NO_SECURITY_CHECK) rc = chk_file_sec(path_jobs, 1, 0, S_IWGRP | S_IWOTH, 1); if (stat(path_users, &statbuf) != 0) (void) mkdir(path_users, 0750); rc |= chk_file_sec(path_users, 1, 0, S_IWGRP | S_IWOTH, 1); rc |= chk_file_sec(path_hooks, 1, 0, S_IWGRP | S_IWOTH, 0); rc |= chk_file_sec(path_hooks_workdir, 1, 0, S_IWGRP | S_IWOTH, 0); rc |= chk_file_sec(path_spool, 1, 1, 0, 0); rc |= chk_file_sec(path_acct, 1, 1, S_IWGRP | S_IWOTH, 0); rc |= chk_file_sec(pbs_conf.pbs_environment, 0, 0, S_IWGRP | S_IWOTH, 1); if (rc) { log_err(-1, __func__, "chk_file_sec has a failure"); return (3); } #endif /* not DEBUG and not NO_SECURITY_CHECK */ time_now = time(NULL); rc = setup_resc(1); if (rc != 0) { /* log_buffer set in setup_resc */ log_err(-1, __func__, log_buffer); /* return value of -1 means a fatal error, -2 means errors * were "auto-corrected" */ if (rc == -1) return (-1); } /* 3. Set default server attibutes values */ memset(&server, 0, sizeof(server)); server.sv_started = time(&time_now); /* time server started */ if (is_sattr_set(SVR_ATR_scheduling)) a_opt = get_sattr_long(SVR_ATR_scheduling); init_server_attrs(); /* 5. If not a "create" initialization, recover server db */ /* and sched db */ rc = svr_recov_db(); if ((rc != 0) && (type != RECOV_CREATE)) { pbs_db_get_errmsg(PBS_DB_ERR, &conn_db_err); if (conn_db_err != NULL) { log_errf(-1, __func__, "%s", conn_db_err); free(conn_db_err); } need_y_response(type, "no server database exists"); type = RECOV_CREATE; } if (type != RECOV_CREATE) { /* Server read success full ?*/ if (rc != 0) { log_errf(rc, __func__, msg_init_baddb); return (-1); } if (is_sattr_set(SVR_ATR_resource_assn)) free_sattr(SVR_ATR_resource_assn); if (new_log_event_mask) { /* set to what was given on command line -e option */ set_sattr_l_slim(SVR_ATR_log_events, new_log_event_mask, SET); } *log_event_mask = get_sattr_long(SVR_ATR_log_events); /* if server comment is a default, clear it */ /* it will be reset as needed */ if (((get_sattr(SVR_ATR_Comment))->at_flags & (ATR_VFLAG_SET | ATR_VFLAG_DEFLT)) == (ATR_VFLAG_SET | ATR_VFLAG_DEFLT)) free_sattr(SVR_ATR_Comment); /* now do sched db */ obj.pbs_db_obj_type = PBS_DB_SCHED; obj.pbs_db_un.pbs_db_sched = &dbsched; rc = pbs_db_search(conn, &obj, NULL, (query_cb_t) &recov_sched_cb); if (rc == -1) { pbs_db_get_errmsg(PBS_DB_ERR, &conn_db_err); if (conn_db_err != NULL) { log_errf(-1, __func__, "%s", conn_db_err); free(conn_db_err); } return (-1); } if (!dflt_scheduler) { dflt_scheduler = sched_alloc(PBS_DFLT_SCHED_NAME); set_sched_default(dflt_scheduler, 0); sched_save_db(dflt_scheduler); } if (get_sattr_long(SVR_ATR_scheduling)) set_scheduler_flag(SCH_SCHEDULE_ETE_ON, NULL); } else { /* init type is "create" */ if (rc == 0) /* server was loaded */ need_y_response(type, "server database exists"); svr_save_db(&server); dflt_scheduler = sched_alloc(PBS_DFLT_SCHED_NAME); set_sched_default(dflt_scheduler, 0); sched_save_db(dflt_scheduler); } /* 4. Check License information */ reset_license_counters(&license_counts); fd = open(path_usedlicenses, O_RDONLY, 0400); if ((fd == -1) || (read(fd, &(license_counts.licenses_high_use), sizeof(pbs_licenses_high_use)) != sizeof(pbs_licenses_high_use))) { license_counts.licenses_high_use.lu_max_hr = 0; license_counts.licenses_high_use.lu_max_day = 0; license_counts.licenses_high_use.lu_max_month = 0; license_counts.licenses_high_use.lu_max_forever = 0; ptm = localtime(&time_now); license_counts.licenses_high_use.lu_day = ptm->tm_mday; license_counts.licenses_high_use.lu_month = ptm->tm_mon; } if (fd != -1) close(fd); set_sattr_str_slim(SVR_ATR_version, PBS_VERSION, NULL); if ((pbs_licensing_location == NULL) && (license_counts.licenses_local == 0)) { printf("%s\n", badlicense); log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_SERVER, LOG_ALERT, msg_daemonname, badlicense); } if (pbs_licensing_location) { sprintf(log_buffer, "Using license server at %s", PBS_LICENSE_LOCATION); log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_SERVER, LOG_NOTICE, msg_daemonname, log_buffer); printf("%s\n", log_buffer); } if (license_counts.licenses_local > 0) { sprintf(log_buffer, "Licenses valid for %ld Floating hosts", license_counts.licenses_local); log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_SERVER, LOG_NOTICE, msg_daemonname, log_buffer); printf("%s\n", log_buffer); } /* start a timed-event every hour to long the number of floating used */ if ((license_counts.licenses_local > 0)) (void) set_task(WORK_Timed, (long) (((time_now + 3600) / 3600) * 3600), call_log_license, 0); /* 6. open accounting file */ if (acct_open(acct_file) != 0) { log_errf(-1, __func__, "Could not open accounting file"); return (-1); } /* 7. Set up other server and global variables */ if (a_opt != -1) { /* a_option was set, overrides saved value of scheduling attr */ set_sattr_l_slim(SVR_ATR_scheduling, a_opt, SET); } /* * 8A. If not a "create" initialization, recover queues. * If a create, remove any queues that might be there. */ if ((queues_idx = pbs_idx_create(0, 0)) == NULL) { log_err(-1, __func__, "Creating queue index failed!"); return (-1); } server.sv_qs.sv_numque = 0; /* get jobs from DB for this instance of server, by port and address */ obj.pbs_db_obj_type = PBS_DB_QUEUE; obj.pbs_db_un.pbs_db_que = &dbque; rc = pbs_db_search(conn, &obj, NULL, (query_cb_t) &recov_queue_cb); if (rc == -1) { pbs_db_get_errmsg(PBS_DB_ERR, &conn_db_err); if (conn_db_err != NULL) { log_errf(-1, __func__, "%s", conn_db_err); free(conn_db_err); } return (-1); } /* Open and read in node list if one exists */ if ((rc = setup_nodes()) == -1) { /* log_buffer set in setup_nodes */ log_errf(-1, __func__, log_buffer); return (-1); } mark_which_queues_have_nodes(); /* at this point, we know all the resource types have been defined, */ /* build the resource summation table for validating the Select directives */ update_resc_sum(); /* * 8B. If not a "create" initialization, recover reservations. */ /* set the zoneinfo directory to $PBS_EXEC/zoneinfo. * This is used for standing reservations user of libical */ sprintf(zone_dir, "%s%s", pbs_conf.pbs_exec_path, ICAL_ZONEINFO_DIR); set_ical_zoneinfo(zone_dir); /* load reservations */ if ((resvs_idx = pbs_idx_create(0, 0)) == NULL) { log_err(-1, __func__, "Creating reservations index failed!"); return (-1); } obj.pbs_db_obj_type = PBS_DB_RESV; obj.pbs_db_un.pbs_db_resv = &dbresv; rc = pbs_db_search(conn, &obj, NULL, (query_cb_t) &recov_resv_cb); if (rc == -1) { pbs_db_get_errmsg(PBS_DB_ERR, &conn_db_err); if (conn_db_err != NULL) { log_errf(-1, __func__, "%s", conn_db_err); free(conn_db_err); } return (-1); } /* * 9. If not "create" or "clean" recovery, recover the jobs. * If a create or clean recovery, delete any jobs. * Before job creation/recovery, create the jobs index. */ if ((jobs_idx = pbs_idx_create(0, 0)) == NULL) { log_err(-1, __func__, "Creating jobs index failed!"); return (-1); } server.sv_qs.sv_numjobs = 0; /* get jobs from DB */ obj.pbs_db_obj_type = PBS_DB_JOB; obj.pbs_db_un.pbs_db_job = &dbjob; rc = pbs_db_search(conn, &obj, NULL, (query_cb_t) &recov_job_cb); if (rc == -1) { pbs_db_get_errmsg(PBS_DB_ERR, &conn_db_err); if (conn_db_err != NULL) { log_errf(-1, __func__, "%s", conn_db_err); free(conn_db_err); } return (-1); } else if (rc == 1) { if ((type != RECOV_CREATE) && (type != RECOV_COLD)) log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_SERVER, LOG_DEBUG, msg_daemonname, msg_init_nojobs); } log_eventf(PBSEVENT_SYSTEM, PBS_EVENTCLASS_SERVER, LOG_NOTICE, msg_daemonname, msg_init_exptjobs, server.sv_qs.sv_numjobs); /* Now, cause any reservations marked RESV_FINISHED to be * removed and place "begin" and "end" tasks onto the * "work_task_timed" list, as appropriate, for those that * remain */ remove_deleted_resvs(); degrade_corrupted_confirmed_resvs(); add_resv_beginEnd_tasks(); resv_timer_init(); /* Put us back in the Server's Private directory */ if (chdir(path_priv) != 0) { (void) sprintf(log_buffer, msg_init_chdir, path_priv); log_err(-1, __func__, log_buffer); return (3); } /* * 10. Recover the hooks. * */ if (chdir(path_hooks) != 0) { (void) sprintf(log_buffer, msg_init_chdir, path_hooks); log_err(errno, __func__, log_buffer); return (-1); } dir = opendir("."); if (dir == NULL) { log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_SERVER, LOG_DEBUG, msg_daemonname, "Could not open hooks dir"); } else { /* Now, for each hook found ... */ while (errno = 0, (pdirent = readdir(dir)) != NULL) { if (chk_save_file(pdirent->d_name) != 0) { continue; } /* recover the hooks */ baselen = strlen(pdirent->d_name) - hook_suf_len; psuffix = pdirent->d_name + baselen; if (strcmp(psuffix, hook_suffix)) { continue; } if ((phook = hook_recov(pdirent->d_name, NULL, hook_msg, sizeof(hook_msg), pbs_python_ext_alloc_python_script, pbs_python_ext_free_python_script)) == NULL) { sprintf(log_buffer, "hook_recov(%s): can't recover - %s", pdirent->d_name, hook_msg); log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_SERVER, LOG_NOTICE, msg_daemonname, log_buffer); } else { sprintf(log_buffer, "Found hook %s type=%s", phook->hook_name, ((phook->type == HOOK_SITE) ? "site" : "pbs")); log_event(PBSEVENT_SYSTEM | PBSEVENT_ADMIN | PBSEVENT_DEBUG, PBS_EVENTCLASS_SERVER, LOG_INFO, msg_daemonname, log_buffer); if (phook->event & MOM_EVENTS) mark_mom_hooks_seen(); } } if (errno != 0 && errno != ENOENT) log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_SERVER, LOG_DEBUG, msg_daemonname, "Could not read hooks dir"); (void) closedir(dir); } print_hooks(0); print_hooks(HOOK_EVENT_QUEUEJOB); print_hooks(HOOK_EVENT_POSTQUEUEJOB); print_hooks(HOOK_EVENT_MODIFYJOB); print_hooks(HOOK_EVENT_RESVSUB); print_hooks(HOOK_EVENT_MODIFYRESV); print_hooks(HOOK_EVENT_MOVEJOB); print_hooks(HOOK_EVENT_RUNJOB); print_hooks(HOOK_EVENT_JOBOBIT); print_hooks(HOOK_EVENT_MANAGEMENT); print_hooks(HOOK_EVENT_MODIFYVNODE); print_hooks(HOOK_EVENT_PROVISION); print_hooks(HOOK_EVENT_PERIODIC); print_hooks(HOOK_EVENT_RESV_CONFIRM); print_hooks(HOOK_EVENT_RESV_BEGIN); print_hooks(HOOK_EVENT_RESV_END); /* * cleanup the hooks work directory */ cleanup_hooks_workdir(0); /* Put us back in the Server's Private directory */ if (chdir(path_priv) != 0) { (void) sprintf(log_buffer, msg_init_chdir, path_priv); log_err(-1, __func__, log_buffer); return (3); } /* 11. Open and read in tracking records */ fd = open(path_track, O_RDONLY | O_CREAT, 0600); if (fd < 0) { log_err(errno, __func__, "unable to open tracking file"); return (-1); } #if !defined(DEBUG) && !defined(NO_SECURITY_CHECK) if (chk_file_sec(path_track, 0, 0, S_IWGRP | S_IWOTH, 0) != 0) return (-1); #endif /* not DEBUG and not NO_SECURITY_CHECK */ if (fstat(fd, &statbuf) < 0) { log_err(errno, "pbs_init", "unable to stat tracking file"); return (-1); } else { size_t amt; size_t rd; char *w; /* validate the size of the file, it should be a multiple */ /* of the tracking structure size */ i = statbuf.st_size / sizeof(struct tracking); amt = i * sizeof(struct tracking); if (amt != statbuf.st_size) { log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_SERVER, LOG_ALERT, msg_daemonname, "tracking file has invalid length"); } if (i < PBS_TRACK_MINSIZE) server.sv_tracksize = PBS_TRACK_MINSIZE; else server.sv_tracksize = i; server.sv_track = (struct tracking *) calloc(server.sv_tracksize, sizeof(struct tracking)); if (server.sv_track == NULL) { log_err(errno, "init", "out of memory"); return -1; } for (i = 0; i < server.sv_tracksize; i++) (server.sv_track + i)->tk_mtime = 0; w = (char *) server.sv_track; /* read in the file (a mutiple of the struct size) */ while (amt > 0) { rd = read(fd, w, amt); if ((rd == -1) && (errno == EINTR)) { continue; } else if (rd <= 0) { break; } amt -= rd; w += rd; } (void) close(fd); server.sv_trackmodifed = 0; } /* set work task to periodically save the tracking records */ (void) set_task(WORK_Timed, (long) (time_now + PBS_SAVE_TRACK_TM), track_save, 0); fd = open(path_prov_track, O_RDONLY | O_CREAT, 0600); if (fd < 0) { log_err(errno, __func__, "unable to open prov_tracking file"); return (-1); } #if !defined(DEBUG) && !defined(NO_SECURITY_CHECK) if (chk_file_sec(path_prov_track, 0, 0, S_IWGRP | S_IWOTH, 0) != 0) return (-1); #endif /* not DEBUG and not NO_SECURITY_CHECK */ if (fstat(fd, &statbuf) < 0) { log_err(errno, "pbs_init", "unable to stat prov_tracking file"); return (-1); } else { size_t amt; size_t rd; char *p, *buffer; /* to hold entire file data */ int ctrl_flag = 0; /* we always write pvtk_mtime first */ char *token; long mtime; i = 0; /* whats the size of data in file */ amt = statbuf.st_size; server.sv_provtracksize = get_sattr_long(SVR_ATR_max_concurrent_prov); DBPRT(("%s: server.sv_provtracksize=%d amt=%ld\n", __func__, server.sv_provtracksize, (long) amt)) p = malloc(amt + 1); if (p == NULL) { log_err(errno, "pbs_init", "unable to malloc"); close(fd); return (-1); } buffer = p; /* read entire file into buffer */ while (amt > 0) { rd = read(fd, p, amt); if ((rd == -1) && (errno == EINTR)) { continue; } else if (rd <= 0) { break; } amt -= rd; p += rd; } (void) close(fd); buffer[statbuf.st_size] = '\0'; server.sv_prov_track = (struct prov_tracking *) calloc(server.sv_provtracksize, sizeof(struct prov_tracking)); if (server.sv_prov_track == NULL) { free(buffer); log_err(errno, "pbs_init", "unable to calloc"); return (-1); } for (i = 0; i < server.sv_provtracksize; i++) { server.sv_prov_track[i].pvtk_mtime = 0; server.sv_prov_track[i].pvtk_pid = -1; server.sv_prov_track[i].pvtk_vnode = NULL; server.sv_prov_track[i].pvtk_aoe_req = NULL; server.sv_prov_track[i].prov_vnode_info = NULL; } /* start tokenizing by '|' */ i = 0; token = strtok(buffer, "|"); while (token != NULL && i < server.sv_provtracksize) { switch (ctrl_flag) { case 0: errno = 0; mtime = strtol(token, NULL, 10); if (errno) { free(buffer); free(server.sv_prov_track); log_err(errno, "pbs_init", "bad data in prov_tracking"); return (-1); } server.sv_prov_track[i].pvtk_mtime = mtime; ++ctrl_flag; break; case 1: /* after first save, 0 is written if */ /* value is null. If reading 0, then */ /* pvtk_vnode should be null else it */ /* becomes "0" */ if (strcmp(token, "0") != 0) { server.sv_prov_track[i].pvtk_vnode = (char *) malloc(strlen(token) + 1); if (server.sv_prov_track[i].pvtk_vnode == NULL) { free(buffer); free(server.sv_prov_track); log_err(errno, "pbs_init", "unable to malloc"); return (-1); } strcpy(server.sv_prov_track[i].pvtk_vnode, token); } ++ctrl_flag; break; case 2: if (strcmp(token, "0") != 0) { server.sv_prov_track[i].pvtk_aoe_req = (char *) malloc(strlen(token) + 1); if (server.sv_prov_track[i].pvtk_vnode == NULL) { free(buffer); free(server.sv_prov_track); log_err(errno, "pbs_init", "unable to malloc"); return (-1); } strcpy(server.sv_prov_track[i].pvtk_aoe_req, token); } ctrl_flag = 0; ++i; break; } token = strtok(NULL, "|"); } server.sv_provtrackmodifed = 0; free(buffer); /* less data recovered than expected */ if ((i != server.sv_provtracksize) && (statbuf.st_size != 0)) { sprintf(log_buffer, "Recovered prov_tracking, " "Expected %d, recovered %d records", server.sv_provtracksize, i); log_event(PBSEVENT_ADMIN, PBS_EVENTCLASS_SERVER, LOG_WARNING, msg_daemonname, log_buffer); } } /* mark all nodes that are in the prov tracking table as offline, * also do away with all jobs that were waiting on such nodes */ offline_all_provisioning_vnodes(); server.sv_cur_prov_records = 0; (void) resize_prov_table(max_concurrent_prov); CLEAR_HEAD(prov_allvnodes); /* trigger degraded reservations on offlined nodes */ degrade_offlined_nodes_reservations(); hook_track_recov(); /* Check to see that jobs in the maintenance_jobs attribute on a node still exist * If they don't exist any more, remove them from a node's maintenance_jobs attribute */ buf = NULL; buf_len = 0; for (i = 0; i < svr_totnodes; i++) { struct array_strings *arst; if (is_nattr_set(pbsndlist[i], ND_ATR_MaintJobs) && (arst = get_nattr_arst(pbsndlist[i], ND_ATR_MaintJobs))->as_usedptr > 0) { int j; int len = 0; int cur_len = 0; attribute new; for (j = 0; j < arst->as_usedptr; j++) len += strlen(arst->as_string[j]) + 1; /* 1 for the comma*/ if (len > buf_len) { char *tmp_buf; tmp_buf = realloc(buf, len + 1); if (tmp_buf == NULL) { free(buf); return (-1); } else { buf = tmp_buf; buf_len = len; } } buf[0] = '\0'; for (j = 0; j < arst->as_usedptr; j++) { if (find_job(arst->as_string[j]) == NULL) { strncat(buf, arst->as_string[j], len); strncat(buf, ",", len); buf[len] = '\0'; } } /* Did we find a string we need to remove*/ cur_len = strlen(buf); if (cur_len > 0) { buf[cur_len - 1] = '\0'; /* remove trailing comma */ clear_attr(&new, &node_attr_def[(int) ND_ATR_MaintJobs]); decode_arst(&new, ATTR_NODE_MaintJobs, NULL, buf); set_arst(get_nattr(pbsndlist[i], ND_ATR_MaintJobs), &new, DECR); } if (arst->as_usedptr > 0) set_vnode_state(pbsndlist[i], INUSE_MAINTENANCE, Nd_State_Or); } } free(buf); /* purge deleted hooks */ phook = (hook *) GET_NEXT(svr_allhooks); while (phook) { phook_current = phook; phook = (hook *) GET_NEXT(phook->hi_allhooks); if (phook_current->pending_delete && !has_pending_mom_action_delete(phook_current->hook_name)) hook_purge(phook_current, pbs_python_ext_free_python_script); } send_rescdef(0); hook_track_save(NULL, -1); /* refresh path_hooks_tracking file */ (void) set_task(WORK_Immed, time_now, memory_debug_log, NULL); return (0); } /** * @brief * reassign_resc - for a recovered running job, reassign the resources and * nodes to the job. * * @param[in,out] pjob - the job. * * @return void */ static void reassign_resc(job *pjob) { int set_exec_vnode; int rc; int unset_resc_released = 0; char *hoststr = get_jattr_str(pjob, JOB_ATR_exec_host); char *hoststr2 = get_jattr_str(pjob, JOB_ATR_exec_host2); char *vnodein; char *vnodeout; /* safety check: if no hoststr, no node (hosts) assigned, just return */ if (hoststr == NULL) return; if ((is_jattr_set(pjob, JOB_ATR_exec_vnode)) == 0) { /* * if exec_vnode is not set, we must be dealing with a * pre-8.0 job. Then we need to set exec_vnode anew based * on the select spec that was auto generated when the job * was requeued and the existing exec_host. This is done in * the same as as when a "qrun -H vn+vn+... jobid" is done. */ set_exec_vnode = 1; vnodein = hoststr; } else { set_exec_vnode = 0; vnodein = get_jattr_str(pjob, JOB_ATR_exec_vnode); } rc = set_nodes((void *) pjob, JOB_OBJECT, vnodein, &vnodeout, &hoststr, &hoststr2, set_exec_vnode, TRUE); if (rc != 0) { sprintf(log_buffer, "Unable to reallocate resources from nodes for job, error %d", rc); log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_JOB, LOG_NOTICE, pjob->ji_qs.ji_jobid, log_buffer); } else if (set_exec_vnode == 1) { /* need to recreate the exec_host/exec_vnode values */ free_jattr(pjob, JOB_ATR_exec_host); free_jattr(pjob, JOB_ATR_exec_vnode); set_jattr_str_slim(pjob, JOB_ATR_exec_vnode, vnodeout, NULL); set_jattr_str_slim(pjob, JOB_ATR_exec_host, hoststr, NULL); } if ((rc == 0) && (is_jattr_set(pjob, JOB_ATR_exec_vnode_deallocated))) { char *hstr = NULL; char *hstr2 = NULL; char *vnalloc = NULL; char *new_exec_vnode_deallocated; new_exec_vnode_deallocated = get_jattr_str(pjob, JOB_ATR_exec_vnode_deallocated); rc = set_nodes((void *) pjob, JOB_OBJECT, new_exec_vnode_deallocated, &vnalloc, &hstr, &hstr2, 1, TRUE); if (rc != 0) { log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, LOG_WARNING, pjob->ji_qs.ji_jobid, "warning: Failed to make some nodes aware of a deleted job"); } } if ((check_job_substate(pjob, JOB_SUBSTATE_SCHSUSP) || check_job_substate(pjob, JOB_SUBSTATE_SUSPEND)) && (is_jattr_set(pjob, JOB_ATR_resc_released))) { /* * Allocating resources back to a suspended job is tricky. * Suspended jobs only hold part of their resources * If set_resc_assigned() is called by a job with the JOB_ATR_resc_released set, * only some of the resources will be acted upon. Since this * is a fresh job from disk, we need to allocate all of * its resources to it before we partially release some. * We do this by temporarily unsetting JOB_ATR_resc_released attribute while * restoring the job's resources. This will allocate all of the * requested resources to the job. We add the flag back to the job * and then decrement the resources released when the job was originally suspended. */ mark_jattr_not_set(pjob, JOB_ATR_resc_released); unset_resc_released = 1; } set_resc_assigned((void *) pjob, 0, INCR); if (unset_resc_released == 1) { mark_jattr_set(pjob, JOB_ATR_resc_released); set_resc_assigned((void *) pjob, 0, DECR); } } /** * @brief * pbsd_init_job - decide what to do with the recovered job structure * * The action depends on the type of initialization. * * @param[in,out] pjob - the job. * @param[in] type - type of initialization. * * @return int * @retval 0 - success * @retval -1 - error. */ int pbsd_init_job(job *pjob, int type) { char newstate; int newsubstate; /* chk if job belongs to a reservation or is a reservation job. If this is true * and the reservation is no longer possible, return (1) else return (0) */ if (Rmv_if_resv_not_possible(pjob)) { account_record(PBS_ACCT_ABT, pjob, ""); svr_mailowner(pjob, MAIL_ABORT, MAIL_NORMAL, msg_init_abt); check_block(pjob, msg_init_abt); job_purge(pjob); return 0; } pjob->ji_momhandle = -1; pjob->ji_mom_prot = PROT_INVALID; /* update at_server attribute in case name changed */ free_jattr(pjob, JOB_ATR_at_server); set_jattr_generic(pjob, JOB_ATR_at_server, server_name, NULL, SET); /* now based on the initialization type */ if ((type == RECOV_COLD) || (type == RECOV_CREATE)) { need_y_response(type, "jobs exists"); init_abt_job(pjob); } else { if (type != RECOV_HOT) pjob->ji_qs.ji_svrflags &= ~JOB_SVFLG_HOTSTART; /* make sure JOB_SVFLG_RescAssn is cleared, */ /* we will reassign resources if needed based on the job's */ /* substate (if the job had resources when server exited */ /* JOB_SVFLG_RescAssn is reset when the resources are */ /* reassigned by calling reassign_resc(). */ pjob->ji_qs.ji_svrflags &= ~JOB_SVFLG_RescAssn; /* Update run_version if it is not set but run_count is, */ /* Likely means recovering a job from a older version */ if ((is_jattr_set(pjob, JOB_ATR_run_version) == 0) && is_jattr_set(pjob, JOB_ATR_runcount) != 0) { set_jattr_l_slim(pjob, JOB_ATR_run_version, get_jattr_long(pjob, JOB_ATR_runcount), SET); } if (pjob->ji_qs.ji_svrflags & JOB_SVFLG_SubJob) { if ((pjob->ji_parentaj = find_arrayparent(pjob->ji_qs.ji_jobid)) == NULL) { /* parent job object not found */ init_abt_job(pjob); return -1; } update_sj_parent(pjob->ji_parentaj, pjob, pjob->ji_qs.ji_jobid, JOB_STATE_LTR_EXPIRED, get_job_state(pjob)); } switch (get_job_substate(pjob)) { case JOB_SUBSTATE_TRANSICM: if (pjob->ji_qs.ji_svrflags & JOB_SVFLG_HERE) { /* * This server created the job, so client * was qsub (a transient client), it won't be * arround to recommit, so auto-commit now */ set_job_state(pjob, JOB_STATE_LTR_QUEUED); set_job_substate(pjob, JOB_SUBSTATE_QUEUED); if (pbsd_init_reque(pjob, CHANGE_STATE) == -1) return -1; } else { /* * another server is sending, append to new job * list and wait for commit; need to clear * receiving sock number though */ pjob->ji_qs.ji_un.ji_newt.ji_fromsock = -1; append_link(&svr_newjobs, &pjob->ji_alljobs, pjob); } break; case JOB_SUBSTATE_TRNOUT: set_job_state(pjob, JOB_STATE_LTR_QUEUED); set_job_substate(pjob, JOB_SUBSTATE_QUEUED); /* requeue as queued */ if (pbsd_init_reque(pjob, CHANGE_STATE) == -1) return -1; break; case JOB_SUBSTATE_TRNOUTCM: if (check_job_state(pjob, JOB_STATE_LTR_RUNNING)) { /* was sending to Mom, requeue for now */ svr_evaljobstate(pjob, &newstate, &newsubstate, 1); svr_setjobstate(pjob, newstate, newsubstate); } else { /* requeue as is - rdy to cmt */ /* resend rtc */ set_task(WORK_Immed, 0, resume_net_move, (void *) pjob); } if (pbsd_init_reque(pjob, KEEP_STATE) == -1) return -1; break; case JOB_SUBSTATE_QUEUED: case JOB_SUBSTATE_PRESTAGEIN: case JOB_SUBSTATE_STAGEIN: case JOB_SUBSTATE_STAGECMP: case JOB_SUBSTATE_STAGEFAIL: case JOB_SUBSTATE_STAGEGO: case JOB_SUBSTATE_HELD: case JOB_SUBSTATE_SYNCHOLD: case JOB_SUBSTATE_DEPNHOLD: case JOB_SUBSTATE_WAITING: if (pbsd_init_reque(pjob, CHANGE_STATE) == -1) return -1; break; case JOB_SUBSTATE_PRERUN: if (pbsd_init_reque(pjob, KEEP_STATE) == -1) return -1; break; case JOB_SUBSTATE_PROVISION: if (is_jattr_set(pjob, JOB_ATR_prov_vnode)) /* If JOB_ATR_prov_vnode is set, free it */ free_jattr(pjob, JOB_ATR_prov_vnode); if (pbsd_init_reque(pjob, CHANGE_STATE) == -1) return -1; break; case JOB_SUBSTATE_RUNNING: case JOB_SUBSTATE_SUSPEND: case JOB_SUBSTATE_SCHSUSP: case JOB_SUBSTATE_BEGUN: if (pbsd_init_reque(pjob, KEEP_STATE) == -1) return -1; if (check_job_substate(pjob, JOB_SUBSTATE_RUNNING) || ((is_jattr_set(pjob, JOB_ATR_resc_released)) && (check_job_substate(pjob, JOB_SUBSTATE_SCHSUSP) || check_job_substate(pjob, JOB_SUBSTATE_SUSPEND)))) { reassign_resc(pjob); if (type == RECOV_HOT) pjob->ji_qs.ji_svrflags |= JOB_SVFLG_HOTSTART; } break; case JOB_SUBSTATE_SYNCRES: /* clear all dependent job ready flags */ if (pbsd_init_reque(pjob, CHANGE_STATE) == -1) return -1; break; case JOB_SUBSTATE_TERM: case JOB_SUBSTATE_EXITING: case JOB_SUBSTATE_STAGEOUT: case JOB_SUBSTATE_STAGEDEL: case JOB_SUBSTATE_EXITED: set_task(WORK_Immed, 0, on_job_exit, (void *) pjob); if (pbsd_init_reque(pjob, KEEP_STATE) == -1) return -1; reassign_resc(pjob); break; case JOB_SUBSTATE_ABORT: /* requeue job and if no nodes assigned,thats all */ if (pbsd_init_reque(pjob, KEEP_STATE) == -1) return -1; if ((pjob->ji_qs.ji_svrflags & JOB_SVFLG_HasNodes) != 0) { /* has nodes so reassign */ set_task(WORK_Immed, 0, on_job_exit, (void *) pjob); reassign_resc(pjob); } break; case JOB_SUBSTATE_MOVED: case JOB_SUBSTATE_FAILED: case JOB_SUBSTATE_FINISHED: case JOB_SUBSTATE_TERMINATED: if (pbsd_init_reque(pjob, KEEP_STATE) == -1) return -1; break; case JOB_SUBSTATE_RERUN: if (check_job_state(pjob, JOB_STATE_LTR_EXITING)) set_task(WORK_Immed, 0, on_job_rerun, (void *) pjob); if (pbsd_init_reque(pjob, KEEP_STATE) == -1) return -1; break; case JOB_SUBSTATE_RERUN1: case JOB_SUBSTATE_RERUN2: case JOB_SUBSTATE_RERUN3: set_task(WORK_Immed, 0, on_job_rerun, (void *) pjob); if (pbsd_init_reque(pjob, KEEP_STATE) == -1) return -1; break; default: (void) sprintf(log_buffer, msg_init_unkstate, get_job_substate(pjob)); log_event(PBSEVENT_ERROR, PBS_EVENTCLASS_JOB, LOG_NOTICE, pjob->ji_qs.ji_jobid, log_buffer); job_abt(pjob, log_buffer); return -1; } /* update entity limit sums for this job */ (void) account_entity_limit_usages(pjob, NULL, NULL, INCR, ETLIM_ACC_ALL); /* if job has exec host of Mom, set addr and port based on hostname */ if (pjob->ji_qs.ji_un_type == JOB_UNION_TYPE_EXEC) { pjob->ji_qs.ji_un.ji_exect.ji_momaddr = 0; pjob->ji_qs.ji_un.ji_exect.ji_momport = 0; if (is_jattr_set(pjob, JOB_ATR_exec_host)) { pbs_net_t new_momaddr; unsigned int new_momport; new_momaddr = get_addr_of_nodebyname( get_jattr_str(pjob, JOB_ATR_exec_host), &new_momport); if (new_momaddr != 0) { pjob->ji_qs.ji_un.ji_exect.ji_momaddr = new_momaddr; pjob->ji_qs.ji_un.ji_exect.ji_momport = new_momport; } else { log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB, LOG_INFO, pjob->ji_qs.ji_jobid, "Failed to update mom address. Mom address not changed."); } } } } return 0; } /** * @brief * pbsd_init_resv - decide what to do with the recovered reservation structure. * * The action depends on the type of initialization. * * @param[in,out] presv - the reservation. * @param[in] type - type of initialization (read-only=0 , or ownership=1) * type is unused for now, will be used in later PRs * */ void pbsd_init_resv(resc_resv *presv, int type) { revert_alter_reservation(presv); is_resv_window_in_future(presv); set_old_subUniverse(presv); /* add resv to server list */ append_link(&svr_allresvs, &presv->ri_allresvs, presv); if (attach_queue_to_reservation(presv)) /* reservation needed queue; failed to find it */ log_eventf(PBSEVENT_SYSTEM | PBSEVENT_ADMIN | PBSEVENT_DEBUG, PBS_EVENTCLASS_RESV, LOG_NOTICE, msg_daemonname, msg_init_resvNOq, presv->ri_qs.ri_queue, presv->ri_qs.ri_resvID); else log_eventf(PBSEVENT_SYSTEM | PBSEVENT_ADMIN | PBSEVENT_DEBUG, PBS_EVENTCLASS_SERVER, LOG_INFO, msg_daemonname, msg_init_recovresv, presv->ri_qs.ri_resvID); } /** * @brief * pbsd_init_node - decide what to do with the recovered node structure. * * The action depends on the type of initialization. * * @param[in,out] dbnode - the node recovered. * @param[in] type - type of initialization (read-only=0 , or ownership=1) * type is unused for now, will be used in later PRs * * @return ptr to pbsnode * @retval Node structure - success * @retval NULL - error. */ struct pbsnode * pbsd_init_node(pbs_db_node_info_t *dbnode, int type) { time_t mom_modtime = 0; struct pbsnode *np; svrattrl *pal; int bad; int rc = 0; int perm = ATR_DFLAG_ACCESS | ATR_PERM_ALLOW_INDIRECT; mom_modtime = dbnode->mom_modtime; pal = GET_NEXT(dbnode->db_attr_list.attrs); /* now create node and subnodes */ rc = create_pbs_node2(dbnode->nd_name, pal, perm, &bad, &np, FALSE, TRUE); /* allow unknown resources */ if (rc) np = NULL; if (np) { if (mom_modtime) np->nd_moms[0]->mi_modtime = mom_modtime; if (is_nattr_set(np, ND_ATR_vnode_pool) && get_nattr_long(np, ND_ATR_vnode_pool) > 0) { mominfo_t *pmom = np->nd_moms[0]; if (pmom && (np == ((mom_svrinfo_t *) (pmom->mi_data))->msr_children[0])) { /* natural vnode being recovered, add to pool */ add_mom_to_pool(np->nd_moms[0]); } } } else { if (rc == PBSE_NODEEXIST) sprintf(log_buffer, "duplicate node \"%s\"", dbnode->nd_name); else sprintf(log_buffer, "could not create node \"%s\", error = %d", dbnode->nd_name, rc); log_errf(-1, __func__, log_buffer); } return np; } /** * @brief * pbsd_init_reque - re-enqueue the job into the queue it was in * * update the state, typically to some form of QUEUED. * make sure substate attributes match actual value. * * @param[in,out] pjob - the job. * @param[in] change_state- possible values, * CHANGE_STATE - 1 * KEEP_STATE - 0 * * @return int * @retval 0 - success * @retval -1 - error. */ static int pbsd_init_reque(job *pjob, int change_state) { char logbuf[384]; char newstate; int newsubstate; int rc; /* re-enqueue the job into the queue it was in */ if (change_state) { /* update the state, typically to some form of QUEUED */ unset_extra_attributes(pjob); svr_evaljobstate(pjob, &newstate, &newsubstate, 1); if (pjob->ji_qs.ji_svrflags & JOB_SVFLG_SubJob) update_sj_parent(pjob->ji_parentaj, pjob, pjob->ji_qs.ji_jobid, get_job_state(pjob), newstate); set_job_state(pjob, newstate); set_job_substate(pjob, newsubstate); } /* make sure substate attributes match actual value */ post_attr_set(get_jattr(pjob, JOB_ATR_substate)); if ((rc = svr_enquejob(pjob, NULL)) == 0) { sprintf(logbuf, msg_init_substate, get_job_substate(pjob)); (void) strcat(logbuf, msg_init_queued); (void) strcat(logbuf, pjob->ji_qs.ji_queue); log_event(PBSEVENT_SYSTEM | PBSEVENT_ADMIN | PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB, LOG_INFO, pjob->ji_qs.ji_jobid, logbuf); } else { if (rc == PBSE_UNKQUE) { /* Oops, this should never happen */ sprintf(logbuf, "%s %s; job %s queue %s", msg_err_noqueue, msg_err_noqueue1, pjob->ji_qs.ji_jobid, pjob->ji_qs.ji_queue); } else if (rc == PBSE_UNKRESC) { sprintf(logbuf, "%s %s; job %s", msg_err_noqueue, msg_unkresc, pjob->ji_qs.ji_jobid); } else { sprintf(logbuf, "%s; job %s queue %s error %d", msg_err_noqueue, pjob->ji_qs.ji_jobid, pjob->ji_qs.ji_queue, rc); } log_err(-1, "pbsd_init", logbuf); (void) job_abt(pjob, logbuf); return (-1); } return (0); } /** * @brief * catch_child() - the signal handler for SIGCHLD. * Set a flag for the main loop to know that a child processes * needs to be reaped. * * @param[in] sig - not used in fun. * * @return void */ static void catch_child(int sig) { extern int reap_child_flag; reap_child_flag = 1; } /** * @brief * change_logs - signal handler for SIGHUP * Causes the accounting file and log file to be closed and reopened. * Thus the old one can be renamed. * * @param[in] sig - not used in fun. * * @return void */ static void change_logs(int sig) { acct_close(); log_close(1); log_open(log_file, path_log); (void) acct_open(acct_file); } /** * @brief * stop_me - signal handler for all caught signals which terminate the server * * Record the signal so an log_event call can be made outside of * the handler, and set the server state to indicate we should shut down. * * @param[in] sig - not used in fun. * * @return void */ /*ARGSUSED*/ static void stop_me(int sig) { set_sattr_l_slim(SVR_ATR_State, SV_STATE_SHUTSIG, SET); } /** * @brief * chk_save_file - check whether data can be saved into file. * * checks include the file permission checks and regular file check. * * @param[in] filename - file which needs to be checked. * * @return error code * @retval 0 - success * @retval -1 - failure */ int chk_save_file(char *filename) { struct stat sb; if (stat(filename, &sb) == -1) return (errno); if (S_ISREG(sb.st_mode)) return (0); return (-1); } /** * @brief * resume_net_move - call net_move() to complete the routing of a job * This is invoked via a work task created on recovery of a job * in JOB_SUBSTATE_TRNOUTCM state. * * @param[in] ptask - work task created on recovery of a job * * @return void */ static void resume_net_move(struct work_task *ptask) { net_move((job *) ptask->wt_parm1, 0); } /** * @brief * need_y_response - on create/clean initialization that would delete * information, obtain the operator approval first. * * @param[in] type - server initialization mode * @param[in] txt - text field in msg_startup3 string * * @return void * * @par MT-safe: No */ static void need_y_response(int type, char *txt) { static int answ = -2; int c; char *t[] = {"Hot", "Warm", "Cold", "Create"}; char *tp; if (answ > 0) return; /* already gotten a response */ fflush(stdin); if ((type > RECOV_CREATE) || (type < RECOV_HOT)) { stop_db(); exit(1); } tp = t[type]; printf(msg_startup3, msg_daemonname, server_name, tp, txt); while (1) { answ = getchar(); c = answ; while ((c != '\n') && (c != EOF)) c = getchar(); switch (answ) { case 'y': case 'Y': return; case EOF: case '\n': case 'n': case 'N': printf("PBS server %s initialization aborted\n", server_name); stop_db(); exit(0); } printf("y(es) or n(o) please:\n"); } } /** * @brief * init_abt_job() - log and email owner message that job is being aborted at * initialization; then purge job (must be called after job is enqueued). * * @param[in] pjob - job * * @return void */ static void init_abt_job(job *pjob) { log_event(PBSEVENT_SYSTEM | PBSEVENT_ADMIN | PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB, LOG_INFO, pjob->ji_qs.ji_jobid, msg_init_abt); svr_mailowner(pjob, MAIL_ABORT, MAIL_NORMAL, msg_init_abt); check_block(pjob, msg_init_abt); job_purge(pjob); } /** * @brief * Rmv_if_resv_not_possible - If the job belongs to a reservation that * is no longer possible then report back that it should not be requeued. * * If the job is in a standing reservation queue then do not check whether it is * viable as this will be handled as part of the end event for the occurrence. * Note that the end event is added to the work task by remove_delete_resvs. * * @param[in,out] pjob - reservation job * * @return return code * @retval 0 - OK to requeue * @retval 1 - should not be requeued */ static int Rmv_if_resv_not_possible(job *pjob) { int rc = 0; /*assume OK to requeue*/ resc_resv *presv; pbs_queue *pque; if ((pque = find_queuebyname(pjob->ji_qs.ji_queue)) != 0) { if ((presv = pque->qu_resvp) != 0) { /*we are dealing with a job in a reservation*/ pjob->ji_myResv = presv; /* If a standing reservation then ignore the check for end time * The behavior of a standing reservation differs from that of an * advance one in that only running jobs are deleted at the end of * an occurrence (be it missed or not). */ if (get_rattr_long(presv, RESV_ATR_resv_count) > 1) return 0; if (presv->ri_qs.ri_etime < time_now) rc = 1; } } return (rc); } /** * @brief * attach_queue_to_reservation - if the reservation happens to * be supported by a pbs_queue, find the queue and attach * it to the reservation * * @param[in,out] presv - reservation. * * @return int * @retval 0 - success * @retval -1 - failure */ static int attach_queue_to_reservation(resc_resv *presv) { if (presv == NULL) return (0); presv->ri_qp = find_queuebyname(presv->ri_qs.ri_queue); if (presv->ri_qp) { /*resv points to queue and queue points back*/ presv->ri_qp->qu_resvp = presv; return (0); } else return (-1); } /** * @brief * call_log_license - call the routine to long the floating license info * * @param[in] ptask - work task structure. * * @return void */ static void call_log_license(struct work_task *ptask) { int fd; long ntime; struct tm *tms; /* log the floating license info */ log_licenses(&license_counts.licenses_high_use); /* reset values for time periods that have passed */ license_counts.licenses_high_use.lu_max_hr = 0; ntime = ptask->wt_event; tms = localtime((time_t *) &ntime); if (tms->tm_mday != license_counts.licenses_high_use.lu_day) { license_counts.licenses_high_use.lu_max_day = 0; license_counts.licenses_high_use.lu_day = tms->tm_mday; } if (tms->tm_mon != license_counts.licenses_high_use.lu_month) { license_counts.licenses_high_use.lu_max_month = 0; license_counts.licenses_high_use.lu_month = tms->tm_mon; } /* write current info to file */ fd = open(path_usedlicenses, O_WRONLY | O_CREAT | O_TRUNC, 0600); if (fd != -1) { if (write(fd, &license_counts.licenses_high_use, sizeof(license_counts.licenses_high_use)) == -1) log_errf(-1, __func__, "write failed. ERR : %s",strerror(errno)); close(fd); } /* call myself again at the top of the next hour */ ntime = ((ntime + 3601) / 3600) * 3600; (void) set_task(WORK_Timed, ntime, call_log_license, 0); }