From 242cebf065b2b709b41b8c987927187141ff8ab5 Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Mon, 1 Sep 2014 16:32:12 +0200 Subject: [PATCH] Log ingest failures and record failures --- src/client.c | 39 +++++++++++++++++++++++++-------------- src/client.h | 4 ++-- src/session.c | 26 ++++++++++++++++++-------- 3 files changed, 45 insertions(+), 24 deletions(-) diff --git a/src/client.c b/src/client.c index d3139a5..723f8be 100644 --- a/src/client.c +++ b/src/client.c @@ -107,6 +107,8 @@ struct client { int record_offset; int show_stat_no; int filtered; /* number of records ignored for local filtering */ + int ingest_failures; /* number of records where XSLT/other failed */ + int record_failures; /* number of records where ZOOM reported error */ int maxrecs; int startrecs; int diagnostic; @@ -577,9 +579,9 @@ void client_search_response(struct client *cl) if (ZOOM_connection_error(link, &error, &addinfo)) { cl->hits = 0; - client_set_state(cl, Client_Error); session_log(se, YLOG_WARN, "%s: Error %s (%s)", client_get_id(cl), error, addinfo); + client_set_state(cl, Client_Error); } else { @@ -630,9 +632,12 @@ static void client_record_ingest(struct client *cl) NMEM nmem = nmem_create(); int rc = ingest_xml_record(cl, xdoc, offset, nmem, 1); if (rc == -1) + { session_log(se, YLOG_WARN, - "Failed to ingest xdoc from %s #%d", + "%s: #%d: failed to ingest xdoc", client_get_id(cl), offset); + cl->ingest_failures++; + } else if (rc == -2) cl->filtered++; nmem_destroy(nmem); @@ -647,6 +652,7 @@ static void client_record_ingest(struct client *cl) { session_log(se, YLOG_WARN, "Record error %s (%s): %s #%d", msg, addinfo, client_get_id(cl), offset); + cl->record_failures++; } else { @@ -662,12 +668,13 @@ static void client_record_ingest(struct client *cl) if (!xmlrec) { const char *rec_syn = ZOOM_record_get(rec, "syntax", NULL); - session_log(se, YLOG_WARN, "ZOOM_record_get failed from %s #%d", + session_log(se, YLOG_WARN, "%s: #%d: ZOOM_record_get failed", client_get_id(cl), offset); session_log(se, YLOG_LOG, "pz:nativesyntax=%s . " "ZOOM record type=%s . Actual record syntax=%s", s ? s : "null", type, rec_syn ? rec_syn : "null"); + cl->ingest_failures++; } else { @@ -677,12 +684,13 @@ static void client_record_ingest(struct client *cl) { const char *rec_syn = ZOOM_record_get(rec, "syntax", NULL); session_log(se, YLOG_WARN, - "Failed to ingest record from %s #%d", + "%s: #%d: failed to ingest record", client_get_id(cl), offset); session_log(se, YLOG_LOG, "pz:nativesyntax=%s . " "ZOOM record type=%s . Actual record syntax=%s", s ? s : "null", type, rec_syn ? rec_syn : "null"); + cl->ingest_failures++; } else if (rc == -2) cl->filtered++; @@ -707,9 +715,10 @@ void client_record_response(struct client *cl, int *got_records) if (ZOOM_connection_error(link, &error, &addinfo)) { + struct session *se = client_get_session(cl); + session_log(se, YLOG_WARN, "%s: Error %s (%s)", + client_get_id(cl), error, addinfo); client_set_state(cl, Client_Error); - yaz_log(YLOG_WARN, "Search error %s (%s): %s", - error, addinfo, client_get_id(cl)); } else { @@ -740,7 +749,7 @@ int client_reingest(struct client *cl) { int i = cl->startrecs; int to = cl->record_offset; - cl->filtered = 0; + cl->record_failures = cl->ingest_failures = cl->filtered = 0; cl->record_offset = i; for (; i < to; i++) @@ -953,7 +962,7 @@ int client_start_search(struct client *cl) session_log(se, YLOG_LOG, "%s: new search", client_get_id(cl)); cl->diagnostic = 0; - cl->filtered = 0; + cl->record_failures = cl->ingest_failures = cl->filtered = 0; client_destroy_xdoc(cl); client_init_xdoc(cl); @@ -1666,16 +1675,18 @@ Odr_int client_get_approximation(struct client *cl) return cl->hits; } -int client_get_num_records(struct client *cl) +int client_get_num_records(struct client *cl, int *filtered, int *ingest, + int *failed) { + if (filtered) + *filtered = cl->filtered; + if (ingest) + *ingest = cl->ingest_failures; + if (failed) + *failed = cl->record_failures; return cl->record_offset; } -int client_get_num_records_filtered(struct client *cl) -{ - return cl->filtered; -} - void client_set_diagnostic(struct client *cl, int diagnostic, const char *message, const char *addinfo) { diff --git a/src/client.h b/src/client.h index cd3e59a..5d8bace 100644 --- a/src/client.h +++ b/src/client.h @@ -87,8 +87,8 @@ int client_parse_query(struct client *cl, const char *query, facet_limits_t facet_limits, const char **error_msg); Odr_int client_get_hits(struct client *cl); Odr_int client_get_approximation(struct client *cl); -int client_get_num_records(struct client *cl); -int client_get_num_records_filtered(struct client *cl); +int client_get_num_records(struct client *cl, int *filtered, int *ingest, + int *failed); int client_get_diagnostic(struct client *cl, const char **message, const char **addinfo); void client_set_diagnostic(struct client *cl, int diagnostic, diff --git a/src/session.c b/src/session.c index e978425..2823399 100644 --- a/src/session.c +++ b/src/session.c @@ -1038,13 +1038,13 @@ static struct hitsbytarget *hitsbytarget_nb(struct session *se, WRBUF w = wrbuf_alloc(); const char *name = session_setting_oneval(client_get_database(cl), PZ_NAME); - res[*count].id = client_get_id(cl); res[*count].name = *name ? name : "Unknown"; res[*count].hits = client_get_hits(cl); res[*count].approximation = client_get_approximation(cl); - res[*count].records = client_get_num_records(cl); - res[*count].filtered = client_get_num_records_filtered(cl); + res[*count].records = client_get_num_records(cl, + &res[*count].filtered, + 0, 0); res[*count].diagnostic = client_get_diagnostic(cl, &res[*count].message, &res[*count].addinfo); @@ -1280,14 +1280,24 @@ int session_fetch_more(struct session *se) } else { - session_log(se, YLOG_LOG, "%s: no more to fetch", - client_get_id(cl)); + int filtered; + int ingest_failures; + int record_failures; + int num = client_get_num_records( + cl, &filtered, &ingest_failures, &record_failures); + session_log(se, YLOG_LOG, "%s: hits=" ODR_INT_PRINTF - " records=%d filtered=%d", + " fetched=%d filtered=%d", client_get_id(cl), client_get_hits(cl), - client_get_num_records(cl), - client_get_num_records_filtered(cl)); + num, filtered); + if (ingest_failures || record_failures) + { + session_log(se, YLOG_WARN, "%s:" + " ingest failures=%d record failures=%d", + client_get_id(cl), + ingest_failures, record_failures); + } } } else -- 1.7.10.4