/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only)
 *
 * ringbuffer/frontend_types.h
 *
 * Ring Buffer Library Synchronization Header (types).
 *
 * Copyright (C) 2010-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * See ring_buffer_frontend.c for more information on wait-free algorithms.
 */

#ifndef _LIB_RING_BUFFER_FRONTEND_TYPES_H
#define _LIB_RING_BUFFER_FRONTEND_TYPES_H

#include <linux/kref.h>
#include <linux/irq_work.h>
#include <ringbuffer/config.h>
#include <ringbuffer/backend_types.h>
#include <lttng/prio_heap.h>	/* For per-CPU read-side iterator */
#include <lttng/cpuhotplug.h>

/*
 * A switch is done during tracing or as a final flush after tracing (so it
 * won't write in the new sub-buffer).
 */
enum switch_mode { SWITCH_ACTIVE, SWITCH_FLUSH };

/* channel-level read-side iterator */
struct channel_iter {
	/* Prio heap of buffers. Lowest timestamps at the top. */
	struct lttng_ptr_heap heap;	/* Heap of struct lttng_kernel_ring_buffer ptrs */
	struct list_head empty_head;	/* Empty buffers linked-list head */
	int read_open;			/* Opened for reading ? */
	u64 last_qs;			/* Last quiescent state timestamp */
	u64 last_timestamp;		/* Last timestamp (for WARN_ON) */
	int last_cpu;			/* Last timestamp cpu */
	/*
	 * read() file operation state.
	 */
	unsigned long len_left;
};

/* channel: collection of per-cpu ring buffers. */
struct lttng_kernel_ring_buffer_channel {
	atomic_t record_disabled;
	unsigned long commit_count_mask;	/*
						 * Commit count mask, removing
						 * the MSBs corresponding to
						 * bits used to represent the
						 * subbuffer index.
						 */

	struct channel_backend backend;		/* Associated backend */

	unsigned long switch_timer_interval;	/* Buffer flush (jiffies) */
	unsigned long read_timer_interval;	/* Reader wakeup (jiffies) */
#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,10,0))
	struct lttng_cpuhp_node cpuhp_prepare;
	struct lttng_cpuhp_node cpuhp_online;
	struct lttng_cpuhp_node cpuhp_iter_online;
#else
	struct notifier_block cpu_hp_notifier;	/* CPU hotplug notifier */
	struct notifier_block hp_iter_notifier;	/* hotplug iterator notifier */
	unsigned int cpu_hp_enable:1;		/* Enable CPU hotplug notif. */
	unsigned int hp_iter_enable:1;		/* Enable hp iter notif. */
#endif
	struct notifier_block tick_nohz_notifier; /* CPU nohz notifier */
	wait_queue_head_t read_wait;		/* reader wait queue */
	wait_queue_head_t hp_wait;		/* CPU hotplug wait queue */
	struct irq_work wakeup_pending;		/* Pending wakeup irq work */
	int finalized;				/* Has channel been finalized */
	struct channel_iter iter;		/* Channel read-side iterator */
	struct kref ref;			/* Reference count */
	int been_active;			/* Tracing was started at least once */
};

/* Per-subbuffer commit counters used on the hot path */
struct commit_counters_hot {
	union v_atomic cc;		/* Commit counter */
	union v_atomic seq;		/* Consecutive commits */
};

/* Per-subbuffer commit counters used only on cold paths */
struct commit_counters_cold {
	union v_atomic cc_sb;		/* Incremented _once_ at sb switch */
};

/* Per-buffer read iterator */
struct lttng_kernel_ring_buffer_iter {
	u64 timestamp;			/* Current record timestamp */
	size_t header_len;		/* Current record header length */
	size_t payload_len;		/* Current record payload length */

	struct list_head empty_node;	/* Linked list of empty buffers */
	unsigned long consumed, read_offset, data_size;
	enum {
		ITER_GET_SUBBUF = 0,
		ITER_TEST_RECORD,
		ITER_NEXT_RECORD,
		ITER_PUT_SUBBUF,
	} state;
	unsigned int allocated:1;
	unsigned int read_open:1;	/* Opened for reading ? */
};

/* ring buffer state */
struct lttng_kernel_ring_buffer {
	/* First 32 bytes cache-hot cacheline */
	union v_atomic offset;		/* Current offset in the buffer */
	struct commit_counters_hot *commit_hot;
					/* Commit count per sub-buffer */
	atomic_long_t consumed;		/*
					 * Current offset in the buffer
					 * standard atomic access (shared)
					 */
	atomic_t record_disabled;
	/* End of first 32 bytes cacheline */
	union v_atomic last_tsc;	/*
					 * Last timestamp written in the buffer.
					 */

	struct lttng_kernel_ring_buffer_backend backend;	/* Associated backend */

	struct commit_counters_cold *commit_cold;
					/* Commit count per sub-buffer */
	u64 *ts_end;			/*
					 * timestamp_end per sub-buffer.
					 * Time is sampled by the
					 * switch_*_end() callbacks which
					 * are the last space reservation
					 * performed in the sub-buffer
					 * before it can be fully
					 * committed and delivered. This
					 * time value is then read by
					 * the deliver callback,
					 * performed by the last commit
					 * before the buffer becomes
					 * readable.
					 */
	atomic_long_t active_readers;	/*
					 * Active readers count
					 * standard atomic access (shared)
					 */
					/* Dropped records */
	union v_atomic records_lost_full;	/* Buffer full */
	union v_atomic records_lost_wrap;	/* Nested wrap-around */
	union v_atomic records_lost_big;	/* Events too big */
	union v_atomic records_count;	/* Number of records written */
	union v_atomic records_overrun;	/* Number of overwritten records */
	wait_queue_head_t read_wait;	/* reader buffer-level wait queue */
	wait_queue_head_t write_wait;	/* writer buffer-level wait queue (for metadata only) */
	struct irq_work wakeup_pending;		/* Pending wakeup irq work */
	int finalized;			/* buffer has been finalized */
	struct timer_list switch_timer;	/* timer for periodical switch */
	struct timer_list read_timer;	/* timer for read poll */
	raw_spinlock_t raw_tick_nohz_spinlock;	/* nohz entry lock/trylock */
	struct lttng_kernel_ring_buffer_iter iter;	/* read-side iterator */
	unsigned long get_subbuf_consumed;	/* Read-side consumed */
	unsigned long prod_snapshot;	/* Producer count snapshot */
	unsigned long cons_snapshot;	/* Consumer count snapshot */
	unsigned int get_subbuf:1,	/* Sub-buffer being held by reader */
		switch_timer_enabled:1,	/* Protected by ring_buffer_nohz_lock */
		read_timer_enabled:1,	/* Protected by ring_buffer_nohz_lock */
		quiescent:1;
};

static inline
void *channel_get_private(struct lttng_kernel_ring_buffer_channel *chan)
{
	return chan->backend.priv;
}

void lib_ring_buffer_lost_event_too_big(struct lttng_kernel_ring_buffer_channel *chan);

/*
 * Issue warnings and disable channels upon internal error.
 * Can receive struct lttng_kernel_ring_buffer or struct lttng_kernel_ring_buffer_backend
 * parameters.
 */
#define CHAN_WARN_ON(c, cond)						\
	({								\
		struct lttng_kernel_ring_buffer_channel *__chan;	\
		int _____ret = unlikely(cond);				\
		if (_____ret) {						\
			if (__same_type(*(c), struct channel_backend))	\
				__chan = container_of((void *) (c),	\
							struct lttng_kernel_ring_buffer_channel, \
							backend);	\
			else if (__same_type(*(c), struct lttng_kernel_ring_buffer_channel)) \
				__chan = (void *) (c);			\
			else						\
				BUG_ON(1);				\
			atomic_inc(&__chan->record_disabled);		\
			WARN_ON(1);					\
		}							\
		_____ret;						\
	})

#endif /* _LIB_RING_BUFFER_FRONTEND_TYPES_H */
