Reload config on change of pihole.toml. This is done using an inotify watcher on /etc/pihole. This also means that there is no need to send SIGHUP to FTL after a config change, this is triggered internally.

Signed-off-by: DL6ER <dl6er@dl6er.de>
This commit is contained in:
DL6ER 2023-01-30 19:05:19 +01:00
parent 4d8a6eddc5
commit 54beca32db
No known key found for this signature in database
GPG Key ID: 00135ACBD90B28DD
21 changed files with 375 additions and 171 deletions

View File

@ -130,7 +130,7 @@ components:
description: Valid session indicator (client is authenticated)
sid:
type: string
description: Session ID (may be `null`)
description: Session ID
nullable: true
validity:
type: integer

View File

@ -181,7 +181,7 @@ components:
type: object
properties:
comment:
description: User-provided free-text comment for this client (may be `null` if not specified)
description: User-provided free-text comment for this client
type: string
nullable: true
default: null

View File

@ -16,7 +16,7 @@ components:
description: "Human-readable error message"
hint:
type: string
description: "Further details (may be `null`)"
description: "Further details"
nullable: true
unauthorized:
type: object
@ -68,4 +68,4 @@ components:
schema:
type: boolean
required: false
example: false
example: false

View File

@ -504,6 +504,8 @@ components:
type: boolean
config:
type: boolean
inotify:
type: boolean
extra:
type: boolean
reserved:
@ -689,6 +691,7 @@ components:
events: false
helper: false
config: false
inotify: false
extra: false
reserved: false
config_one:

View File

@ -248,7 +248,7 @@ components:
type: object
properties:
comment:
description: User-provided free-text comment for this domain (may be `null` if not specified)
description: User-provided free-text comment for this domain
type: string
nullable: true
default: null

View File

@ -175,7 +175,7 @@ components:
type: object
properties:
comment:
description: User-provided free-text comment for this group (may be `null` if not specified)
description: User-provided free-text comment for this group
type: string
nullable: true
default: null

View File

@ -160,7 +160,7 @@ components:
name:
type: string
nullable: true
description: Client name (may be `null`)
description: Client name
ip:
type: string
description: Client IP address

View File

@ -184,7 +184,7 @@ components:
type: object
properties:
comment:
description: User-provided free-text comment for this list (may be `null` if not specified)
description: User-provided free-text comment for this list
type: string
nullable: true
default: null

View File

@ -167,25 +167,25 @@ components:
description: Queried domain
cname:
type: string
description: Domain blocked during deep CNAME inspection (may be `null`)
description: Domain blocked during deep CNAME inspection
nullable: true
status:
type: string
description: Query status (may be `null`)
description: Query status
nullable: true
client:
type: string
description: Requesting client (may be an IP address or a hostname)
dnssec:
type: string
description: DNSSEC status (may be `null`)
description: DNSSEC status
nullable: true
reply:
type: object
properties:
type:
type: string
description: Reply type (may be `null`)
description: Reply type
nullable: true
time:
type: number
@ -198,7 +198,7 @@ components:
description: ID of blocking regex (`-1` if N/A)
upstream:
type: string
description: IP or name + port of upstream server (may be `null`)
description: IP or name + port of upstream server
nullable: true
dbid:
type: integer
@ -288,4 +288,4 @@ components:
type: array
description: Array of suggested DNSSEC statuses
items:
type: string
type: string

View File

@ -45,7 +45,7 @@ components:
example: "blockeddomain.com"
comment:
type: string
description: Optional comment (may be `null`)
description: Optional comment
nullable: true
example: "I needed to block this because of XYZ"
enabled:
@ -84,7 +84,7 @@ components:
example: "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts"
comment:
type: string
description: Optional comment of the adlist (may be `null`)
description: Optional comment of the adlist
nullable: true
example: "I needed to block this because of XYZ"
enabled:
@ -122,7 +122,7 @@ components:
example: "^something[0-9*];querytype=A"
comment:
type: string
description: Optional comment (may be `null`)
description: Optional comment
nullable: true
example: "I want to allow anything starting in something0 (but also all other digits) when it is an A query"
enabled:
@ -157,7 +157,7 @@ components:
example: ".;querytype=ANY"
comment:
type: string
description: Optional comment (may be `null`)
description: Optional comment
nullable: true
example: "Blocking all ANY queries"
enabled:

View File

@ -15,6 +15,8 @@ set(sources
config.h
dnsmasq_config.c
dnsmasq_config.h
inotify.c
inotify.h
legacy_reader.c
legacy_reader.h
toml_writer.c

View File

@ -1091,6 +1091,12 @@ void initConfig(struct config *conf)
conf->debug.config.f = FLAG_ADVANCED_SETTING;
conf->debug.config.d.b = false;
conf->debug.inotify.k = "debug.inotify";
conf->debug.inotify.h = "Debug monitoring of /etc/pihole filesystem events";
conf->debug.inotify.t = CONF_BOOL;
conf->debug.inotify.f = FLAG_ADVANCED_SETTING;
conf->debug.inotify.d.b = false;
conf->debug.extra.k = "debug.extra";
conf->debug.extra.h = "Temporary flag that may print additional information. This debug flag is meant to be used whenever needed for temporary investigations. The logged content may change without further notice at any time.";
conf->debug.extra.t = CONF_BOOL;

View File

@ -264,6 +264,7 @@ struct config {
struct conf_item events;
struct conf_item helper;
struct conf_item config;
struct conf_item inotify;
struct conf_item extra;
struct conf_item reserved;
} debug;

126
src/config/inotify.c Normal file
View File

@ -0,0 +1,126 @@
/* Pi-hole: A black hole for Internet advertisements
* (c) 2023 Pi-hole, LLC (https://pi-hole.net)
* Network-wide ad blocking via your own hardware.
*
* FTL Engine
* Config inotify routines
*
* This file is copyright under the latest version of the EUPL.
* Please see LICENSE file for your rights under this license. */
#include "config/inotify.h"
#include "log.h"
#include <sys/inotify.h>
// NAME_MAX
#include <limits.h>
#define WATCHDIR "/etc/pihole"
static int inotify_fd = -1;
static int inotify_wd = -1;
static bool create_inotify_watcher(void)
{
// Create inotify instance
inotify_fd = inotify_init1(IN_NONBLOCK);
if(inotify_fd == -1)
{
log_warn("Cannot create inotify instance: %s", strerror(errno));
return false;
}
// Add watch to inotify instance
// We are interested in the following events:
// - IN_CREATE: File was created
// - IN_CLOSE_WRITE: File was closed after writing
// - IN_MOVE: File was moved
// - IN_DELETE: File was deleted
// - IN_ONLYDIR: Race-free check of ensuring that the monitored object is a directory
inotify_wd = inotify_add_watch(inotify_fd, WATCHDIR, IN_CREATE | IN_CLOSE_WRITE | IN_MOVE | IN_DELETE | IN_ONLYDIR);
if(inotify_wd == -1)
{
log_warn("Cannot add watching of "WATCHDIR" to inotify instance: %s", strerror(errno));
return false;
}
log_debug(DEBUG_INOTIFY, "Created inotify watcher for "WATCHDIR);
return true;
}
static void close_inotify_watcher(void)
{
// Harmless no-op, this happens at the first refresh of dnsmasq
if(inotify_fd == -1 && inotify_wd == -1)
return;
// Remove watch from inotify instance
if(inotify_rm_watch(inotify_fd, inotify_wd) == -1)
log_warn("Cannot remove watch from inotify instance: %s", strerror(errno));
inotify_wd = -1;
// Close inotify instance
if(close(inotify_fd) == -1)
log_warn("Cannot close inotify instance: %s", strerror(errno));
inotify_fd = -1;
log_debug(DEBUG_INOTIFY, "Closed inotify watcher");
}
void watch_config(bool watch)
{
// Set global variable
if(watch)
create_inotify_watcher();
else
close_inotify_watcher();
}
bool check_inotify_event(void)
{
// Check if we are watching for changes
if(inotify_fd == -1 || inotify_wd == -1)
return false;
// Read inotify events (if any)
// The buffer size is chosen to be large enough to read at least ten events
char buf[10*(sizeof(struct inotify_event) + NAME_MAX + 1)];
const ssize_t len = read(inotify_fd, buf, sizeof(buf));
if(len == -1 && errno != EAGAIN)
{
log_err("Cannot read inotify events: %s", strerror(errno));
return false;
}
// Process all events
void *ptr;
bool config_changed = false;
const struct inotify_event *event;
for (ptr = buf; ptr < (void*)buf + len; ptr += sizeof(struct inotify_event) + event->len)
{
event = (const struct inotify_event *) ptr;
// Check if this is the correct watch descriptor
if(event->wd != inotify_wd)
continue;
// Check if this is the event we are looking for
if(event->mask & IN_CLOSE_WRITE)
{
log_debug(DEBUG_INOTIFY, "File written: "WATCHDIR"/%s", event->name);
if(event->name != NULL && strcmp(event->name, "pihole.toml") == 0)
config_changed = true;
}
else if(event->mask & IN_CREATE)
log_debug(DEBUG_INOTIFY, "File created: "WATCHDIR"/%s", event->name);
else if(event->mask & IN_MOVE)
log_debug(DEBUG_INOTIFY, "File moved: "WATCHDIR"/%s", event->name);
else if(event->mask & IN_DELETE)
log_debug(DEBUG_INOTIFY, "File deleted: "WATCHDIR"/%s", event->name);
else if(event->mask & IN_IGNORED)
log_warn("Inotify watch descriptor for "WATCHDIR" was removed (directory deleted or unmounted?)");
else
log_debug(DEBUG_INOTIFY, "Unknown event (%X) on watched file: "WATCHDIR"/%s", event->mask, event->name);
}
return config_changed;
}

20
src/config/inotify.h Normal file
View File

@ -0,0 +1,20 @@
/* Pi-hole: A black hole for Internet advertisements
* (c) 2023 Pi-hole, LLC (https://pi-hole.net)
* Network-wide ad blocking via your own hardware.
*
* FTL Engine
* Config inotify prototypes
*
* This file is copyright under the latest version of the EUPL.
* Please see LICENSE file for your rights under this license. */
#ifndef CONFIG_INOTIFY_H
#define CONFIG_INOTIFY_H
#include <stdbool.h>
#include <string.h>
#include <errno.h>
#include <unistd.h>
void watch_config(bool watch);
bool check_inotify_event(void);
#endif //CONFIG_INOTIFY_H

View File

@ -17,14 +17,21 @@
#include "toml_helper.h"
// get_blocking_mode_str()
#include "datastructure.h"
// watch_config()
#include "config/inotify.h"
bool writeFTLtoml(const bool verbose)
{
// Stop watching for changes in the config file
watch_config(false);
// Try to open global config file
FILE *fp;
if((fp = openFTLtoml("w")) == NULL)
{
log_warn("Cannot write to FTL config file, content not updated");
// Restart watching for changes in the config file
watch_config(true);
return false;
}
@ -104,5 +111,8 @@ bool writeFTLtoml(const bool verbose)
// Close file and release exclusive lock
closeFTLtoml(fp);
// Restart watching for changes in the config file
watch_config(true);
return true;
}

View File

@ -3310,7 +3310,7 @@ int check_struct_sizes(void)
// sizeof(struct conf_item) is 72 on x86_64 and 52 on x86_32
// number of config elements: CONFIG_ELEMENTS
result += check_one_struct("struct conf_item", sizeof(struct conf_item), 72, 52);
result += check_one_struct("struct config", sizeof(struct config), 8280, 5980);
result += check_one_struct("struct config", sizeof(struct config), 8352, 6032);
result += check_one_struct("queriesData", sizeof(queriesData), 72, 64);
result += check_one_struct("upstreamsData", sizeof(upstreamsData), 640, 628);
result += check_one_struct("clientsData", sizeof(clientsData), 672, 652);

View File

@ -154,6 +154,7 @@ enum debug_flag {
DEBUG_EVENTS,
DEBUG_HELPER,
DEBUG_CONFIG,
DEBUG_INOTIFY,
DEBUG_EXTRA,
DEBUG_RESERVED,
DEBUG_MAX
@ -240,6 +241,7 @@ enum thread_types {
DB,
GC,
DNSclient,
CONF_READER,
THREADS_MAX
} __attribute__ ((packed));

332
src/gc.c
View File

@ -30,6 +30,8 @@
#include "files.h"
// void calc_cpu_usage()
#include "daemon.h"
// create_inotify_watcher()
#include "config/inotify.h"
// Resource checking interval
// default: 300 seconds
@ -118,6 +120,162 @@ static void check_load(void)
log_resource_shortage(load[2], nprocs, -1, -1, NULL, NULL);
}
static void runGC(const time_t now, time_t *lastGCrun)
{
doGC = false;
// Update lastGCrun timer
*lastGCrun = now - GCdelay - (now - GCdelay)%GCinterval;
// Lock FTL's data structure, since it is likely that it will be changed here
// Requests should not be processed/answered when data is about to change
lock_shm();
// Get minimum timestamp to keep (this can be set with MAXLOGAGE)
time_t mintime = (now - GCdelay) - config.database.maxHistory.v.ui;
// Align the start time of this GC run to the GCinterval. This will also align with the
// oldest overTime interval after GC is done.
mintime -= mintime % GCinterval;
if(config.debug.gc.v.b)
{
timer_start(GC_TIMER);
char timestring[TIMESTR_SIZE] = "";
get_timestr(timestring, mintime, false, false);
log_info("GC starting, mintime: %s (%lu)", timestring, (unsigned long)mintime);
}
// Process all queries
int removed = 0;
for(long int i=0; i < counters->queries; i++)
{
queriesData* query = getQuery(i, true);
if(query == NULL)
continue;
// Test if this query is too new
if(query->timestamp > mintime)
break;
// Adjust client counter (total and overTime)
clientsData* client = getClient(query->clientID, true);
const int timeidx = getOverTimeID(query->timestamp);
overTime[timeidx].total--;
if(client != NULL)
change_clientcount(client, -1, 0, timeidx, -1);
// Adjust domain counter (no overTime information)
domainsData* domain = getDomain(query->domainID, true);
if(domain != NULL)
domain->count--;
// Get upstream pointer
// Change other counters according to status of this query
switch(query->status)
{
case QUERY_UNKNOWN:
// Unknown (?)
break;
case QUERY_FORWARDED: // (fall through)
case QUERY_RETRIED: // (fall through)
case QUERY_RETRIED_DNSSEC:
// Forwarded to an upstream DNS server
// Adjusting counters is done below in moveOverTimeMemory()
break;
case QUERY_CACHE:
case QUERY_CACHE_STALE:
// Answered from local cache _or_ local config
break;
case QUERY_GRAVITY: // Blocked by Pi-hole's blocking lists (fall through)
case QUERY_DENYLIST: // Exact blocked (fall through)
case QUERY_REGEX: // Regex blocked (fall through)
case QUERY_EXTERNAL_BLOCKED_IP: // Blocked by upstream provider (fall through)
case QUERY_EXTERNAL_BLOCKED_NXRA: // Blocked by upstream provider (fall through)
case QUERY_EXTERNAL_BLOCKED_NULL: // Blocked by upstream provider (fall through)
case QUERY_GRAVITY_CNAME: // Gravity domain in CNAME chain (fall through)
case QUERY_REGEX_CNAME: // Regex denied domain in CNAME chain (fall through)
case QUERY_DENYLIST_CNAME: // Exactly denied domain in CNAME chain (fall through)
case QUERY_DBBUSY: // Blocked because gravity database was busy
case QUERY_SPECIAL_DOMAIN: // Blocked by special domain handling
//counters->blocked--;
overTime[timeidx].blocked--;
if(domain != NULL)
domain->blockedcount--;
if(client != NULL)
change_clientcount(client, 0, -1, -1, 0);
break;
case QUERY_IN_PROGRESS: // Don't have to do anything here
case QUERY_STATUS_MAX: // fall through
default:
/* That cannot happen */
break;
}
// Update reply countersthread_running[GC] = false;
counters->reply[query->reply]--;
// Update type counters
if(query->type < TYPE_MAX)
counters->querytype[query->type]--;
// Subtract UNKNOWN from the counters before
// setting the status if different. This ensure
// we are not counting them at all.
if(query->status != QUERY_UNKNOWN)
counters->status[QUERY_UNKNOWN]--;
// Set query again to UNKNOWN to reset the counters
query_set_status(query, QUERY_UNKNOWN);
// Count removed queries
removed++;
// Remove query from queries table (temp), we
// can release the lock for this action to
// prevent blocking the DNS service too long
unlock_shm();
delete_query_from_db(query->db);
lock_shm();
}
// Only perform memory operations when we actually removed queries
if(removed > 0)
{
// Move memory forward to keep only what we want
// Note: for overlapping memory blocks, memmove() is a safer approach than memcpy()
// Example: (I = now invalid, X = still valid queries, F = free space)
// Before: IIIIIIXXXXFF
// After: XXXXFFFFFFFF
queriesData *dest = getQuery(0, true);
queriesData *src = getQuery(removed, true);
if(dest && src)
memmove(dest, src, (counters->queries - removed)*sizeof(queriesData));
// Update queries counter
counters->queries -= removed;
// ensure remaining memory is zeroed out (marked as "F" in the above example)
queriesData *tail = getQuery(counters->queries, true);
if(tail)
memset(tail, 0, (counters->queries_MAX - counters->queries)*sizeof(queriesData));
}
// Determine if overTime memory needs to get moved
moveOverTimeMemory(mintime);
log_debug(DEBUG_GC, "GC removed %i queries (took %.2f ms)", removed, timer_elapsed_msec(GC_TIMER));
// Release thread lock
unlock_shm();
// After storing data in the database for the next time,
// we should scan for old entries, which will then be deleted
// to free up pages in the database and prevent it from growing
// ever larger and larger
DBdeleteoldqueries = true;
}
void *GC_thread(void *val)
{
// Set thread name
@ -134,6 +292,9 @@ void *GC_thread(void *val)
unsigned int LastLogStorageUsage = 0;
unsigned int LastDBStorageUsage = 0;
// Create inotify watcher for pihole.toml config file
watch_config(true);
// Run as long as this thread is not canceled
while(!killed)
{
@ -164,164 +325,31 @@ void *GC_thread(void *val)
lastResourceCheck = now;
}
// Intermediate cancellation-point
if(killed)
break;
if(now - GCdelay - lastGCrun >= GCinterval || doGC)
runGC(now, &lastGCrun);
// Intermediate cancellation-point
if(killed)
break;
// Check if pihole.toml has been modified
if(check_inotify_event())
{
doGC = false;
// Update lastGCrun timer
lastGCrun = now - GCdelay - (now - GCdelay)%GCinterval;
// Lock FTL's data structure, since it is likely that it will be changed here
// Requests should not be processed/answered when data is about to change
lock_shm();
// Get minimum timestamp to keep (this can be set with MAXLOGAGE)
time_t mintime = (now - GCdelay) - config.database.maxHistory.v.ui;
// Align the start time of this GC run to the GCinterval. This will also align with the
// oldest overTime interval after GC is done.
mintime -= mintime % GCinterval;
if(config.debug.gc.v.b)
{
timer_start(GC_TIMER);
char timestring[TIMESTR_SIZE] = "";
get_timestr(timestring, mintime, false, false);
log_info("GC starting, mintime: %s (%lu)", timestring, (unsigned long)mintime);
}
// Process all queries
int removed = 0;
for(long int i=0; i < counters->queries; i++)
{
queriesData* query = getQuery(i, true);
if(query == NULL)
continue;
// Test if this query is too new
if(query->timestamp > mintime)
break;
// Adjust client counter (total and overTime)
clientsData* client = getClient(query->clientID, true);
const int timeidx = getOverTimeID(query->timestamp);
overTime[timeidx].total--;
if(client != NULL)
change_clientcount(client, -1, 0, timeidx, -1);
// Adjust domain counter (no overTime information)
domainsData* domain = getDomain(query->domainID, true);
if(domain != NULL)
domain->count--;
// Get upstream pointer
// Change other counters according to status of this query
switch(query->status)
{
case QUERY_UNKNOWN:
// Unknown (?)
break;
case QUERY_FORWARDED: // (fall through)
case QUERY_RETRIED: // (fall through)
case QUERY_RETRIED_DNSSEC:
// Forwarded to an upstream DNS server
// Adjusting counters is done below in moveOverTimeMemory()
break;
case QUERY_CACHE:
case QUERY_CACHE_STALE:
// Answered from local cache _or_ local config
break;
case QUERY_GRAVITY: // Blocked by Pi-hole's blocking lists (fall through)
case QUERY_DENYLIST: // Exact blocked (fall through)
case QUERY_REGEX: // Regex blocked (fall through)
case QUERY_EXTERNAL_BLOCKED_IP: // Blocked by upstream provider (fall through)
case QUERY_EXTERNAL_BLOCKED_NXRA: // Blocked by upstream provider (fall through)
case QUERY_EXTERNAL_BLOCKED_NULL: // Blocked by upstream provider (fall through)
case QUERY_GRAVITY_CNAME: // Gravity domain in CNAME chain (fall through)
case QUERY_REGEX_CNAME: // Regex denied domain in CNAME chain (fall through)
case QUERY_DENYLIST_CNAME: // Exactly denied domain in CNAME chain (fall through)
case QUERY_DBBUSY: // Blocked because gravity database was busy
case QUERY_SPECIAL_DOMAIN: // Blocked by special domain handling
//counters->blocked--;
overTime[timeidx].blocked--;
if(domain != NULL)
domain->blockedcount--;
if(client != NULL)
change_clientcount(client, 0, -1, -1, 0);
break;
case QUERY_IN_PROGRESS: // Don't have to do anything here
case QUERY_STATUS_MAX: // fall through
default:
/* That cannot happen */
break;
}
// Update reply countersthread_running[GC] = false;
counters->reply[query->reply]--;
// Update type counters
if(query->type < TYPE_MAX)
counters->querytype[query->type]--;
// Subtract UNKNOWN from the counters before
// setting the status if different. This ensure
// we are not counting them at all.
if(query->status != QUERY_UNKNOWN)
counters->status[QUERY_UNKNOWN]--;
// Set query again to UNKNOWN to reset the counters
query_set_status(query, QUERY_UNKNOWN);
// Count removed queries
removed++;
// Remove query from queries table (temp), we
// can release the lock for this action to
// prevent blocking the DNS service too long
unlock_shm();
delete_query_from_db(query->db);
lock_shm();
}
// Only perform memory operations when we actually removed queries
if(removed > 0)
{
// Move memory forward to keep only what we want
// Note: for overlapping memory blocks, memmove() is a safer approach than memcpy()
// Example: (I = now invalid, X = still valid queries, F = free space)
// Before: IIIIIIXXXXFF
// After: XXXXFFFFFFFF
queriesData *dest = getQuery(0, true);
queriesData *src = getQuery(removed, true);
if(dest && src)
memmove(dest, src, (counters->queries - removed)*sizeof(queriesData));
// Update queries counter
counters->queries -= removed;
// ensure remaining memory is zeroed out (marked as "F" in the above example)
queriesData *tail = getQuery(counters->queries, true);
if(tail)
memset(tail, 0, (counters->queries_MAX - counters->queries)*sizeof(queriesData));
}
// Determine if overTime memory needs to get moved
moveOverTimeMemory(mintime);
log_debug(DEBUG_GC, "GC removed %i queries (took %.2f ms)", removed, timer_elapsed_msec(GC_TIMER));
// Release thread lock
unlock_shm();
// After storing data in the database for the next time,
// we should scan for old entries, which will then be deleted
// to free up pages in the database and prevent it from growing
// ever larger and larger
DBdeleteoldqueries = true;
// Reload config
log_info("Reloading config due to pihole.toml change");
kill(0, SIGHUP);
}
thread_sleepms(GC, 1000);
}
// Close inotify watcher
watch_config(false);
log_info("Terminating GC thread");
thread_running[GC] = false;
return NULL;

View File

@ -247,6 +247,9 @@ void debugstr(const enum debug_flag flag, const char **name)
case DEBUG_CONFIG:
*name = "DEBUG_CONFIG";
return;
case DEBUG_INOTIFY:
*name = "DEBUG_INOTIFY";
return;
case DEBUG_RESERVED:
*name = "DEBUG_RESERVED";
return;

View File

@ -763,6 +763,9 @@
# Print config parsing details
config = true ### CHANGED, default = false
# Debug monitoring of /etc/pihole filesystem events
inotify = true ### CHANGED, default = false
# Temporary flag that may print additional information. This debug flag is meant to be
# used whenever needed for temporary investigations. The logged content may change
# without further notice at any time.