[ovs-dev] [RFC PATCH 10/21] keepalive: Retrieve PMD status periodically.

[Bhanuprakash Bodireddy]

This commit implements APIs to retrieve the PMD thread status and return
the status in the below format for each PMD thread.

  Format: CORE_<id>="STATUS, last_seen_timestamp"
  eg:     CORE_1="ALIVE,9220698256784207"
          CORE_2="GONE,9220698256786231"

The status is periodically retrieved by keepalive thread and stored in
keepalive_stats struc which later shall be retrieved by vswitchd thread.

Also the datapath status is updated by considering the status of all
active PMD threads. In case of four PMD threads the status is as below:

   "CORE_0="ALIVE,9220698256784207"
    CORE_1="ALIVE,9220698256784913"
    CORE_2="ALIVE,9220698256785902"
    CORE_3="ALIVE,9220698256786231"
    Datapath status:HEALTHY"

Signed-off-by: Bhanuprakash Bodireddy <bhanuprakash.bodireddy at intel.com>
---
 lib/dpif-netdev.c |  1 +
 lib/keepalive.c   | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 lib/keepalive.h   |  1 +
 3 files changed, 83 insertions(+)

diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c
index 97c3807..c76c74c 100644
--- a/lib/dpif-netdev.c
+++ b/lib/dpif-netdev.c
@@ -981,6 +981,7 @@ ovs_keepalive(void *f_)
         int n_pmds = cmap_count(&dp->poll_threads) - 1;
         if (n_pmds > 0) {
             dispatch_heartbeats();
+            get_ka_stats();
         }
 
         ovsrcu_quiesce_start();
diff --git a/lib/keepalive.c b/lib/keepalive.c
index 33ddd00..239c666 100644
--- a/lib/keepalive.c
+++ b/lib/keepalive.c
@@ -25,8 +25,10 @@
 #include "keepalive.h"
 #include "lib/vswitch-idl.h"
 #include "openvswitch/vlog.h"
+#include "ovs-thread.h"
 
 VLOG_DEFINE_THIS_MODULE(keepalive);
+static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 20);
 
 static bool keepalive_enable = false;   /* KeepAlive disabled by default */
 static bool ka_init_status = ka_init_failure; /* KeepAlive initialization */
@@ -35,6 +37,9 @@ static uint32_t keepalive_timer_interval;     /* keepalive timer interval */
 static const char *keepalive_shm_blk = NULL;
 struct keepalive_shm *ka_shm = NULL;
 
+static struct ovs_mutex mutex = OVS_MUTEX_INITIALIZER;
+static struct smap *keepalive_stats OVS_GUARDED_BY(mutex);
+
 /* Return the Keepalive shared memory block name. */
 static inline const char *
 get_ka_shm_blk(void)
@@ -175,6 +180,82 @@ struct keepalive_shm *keepalive_shm_create(void)
     return NULL;
 }
 
+static void
+get_pmd_status(struct smap *ka_pmd_stats)
+{
+    struct keepalive_shm *ka_shm = get_ka_shm();
+    if (!ka_shm) {
+        VLOG_ERR_RL(&rl, "KeepAlive: Invalid shared memory block.");
+        return;
+    }
+
+    int datapath_failure = 0;
+    int n_cores = count_cpu_cores();
+    for (int core_id = 0; core_id < n_cores; core_id++) {
+        char *state = NULL;
+        char *core_id_str;
+
+        if (ka_shm->core_state[core_id] == KA_STATE_UNUSED ||
+            ka_shm->core_state[core_id] == KA_STATE_SLEEP ) {
+            continue;
+        }
+
+        switch (ka_shm->core_state[core_id]) {
+        case KA_STATE_ALIVE:
+            state = "ALIVE";
+            break;
+        case KA_STATE_MISSING:
+            state = "MISSING";
+            break;
+        case KA_STATE_DEAD:
+            state = "DEAD";
+            break;
+        case KA_STATE_GONE:
+            state = "GONE";
+            datapath_failure++;
+            break;
+        case KA_STATE_DOZING:
+            state = "DOZING";
+            break;
+        case KA_STATE_SLEEP:
+            state = "SLEEP";
+            break;
+        case KA_STATE_CHECK:
+            state = "HEALTH_CHECK_RUNNING";
+            break;
+        case KA_STATE_UNUSED:
+            break;
+        }
+
+        core_id_str = xasprintf("CORE_%d", core_id);
+        smap_add_format(ka_pmd_stats, core_id_str, "%s,%ld",
+                        state, ka_shm->core_last_seen_times[core_id]);
+        free(core_id_str);
+    }
+
+    smap_add_format(ka_pmd_stats, "Datapath status", "%s",
+                           datapath_failure ? "BAD" : "HEALTHY");
+}
+
+void
+get_ka_stats(void)
+{
+    struct smap *ka_pmd_stats;
+    ka_pmd_stats = xmalloc(sizeof *ka_pmd_stats);
+    smap_init(ka_pmd_stats);
+
+    get_pmd_status(ka_pmd_stats);
+
+    ovs_mutex_lock(&mutex);
+    if (keepalive_stats) {
+        smap_destroy(keepalive_stats);
+        free(keepalive_stats);
+        keepalive_stats = NULL;
+    }
+    keepalive_stats = ka_pmd_stats;
+    ovs_mutex_unlock(&mutex);
+}
+
 static int
 ka_init__(void)
 {
diff --git a/lib/keepalive.h b/lib/keepalive.h
index 920e934..bdee16e 100644
--- a/lib/keepalive.h
+++ b/lib/keepalive.h
@@ -64,5 +64,6 @@ void ka_mark_pmd_thread_sleep(void);
 bool is_ka_enabled(void);
 uint32_t get_ka_interval(void);
 int get_ka_init_status(void);
+void get_ka_stats(void);
 
 #endif /* keepalive.h */
-- 
2.4.11