2017-03-15 01:46:17 +08:00
|
|
|
/* Pi-hole: A black hole for Internet advertisements
|
|
|
|
* (c) 2017 Pi-hole, LLC (https://pi-hole.net)
|
|
|
|
* Network-wide ad blocking via your own hardware.
|
|
|
|
*
|
|
|
|
* FTL Engine
|
|
|
|
* Garbage collection routines
|
|
|
|
*
|
|
|
|
* This file is copyright under the latest version of the EUPL.
|
|
|
|
* Please see LICENSE file for your rights under this license. */
|
|
|
|
|
|
|
|
#include "FTL.h"
|
2019-06-30 00:02:14 +08:00
|
|
|
#include "gc.h"
|
2018-10-05 13:18:40 +08:00
|
|
|
#include "shmem.h"
|
2019-06-30 03:44:01 +08:00
|
|
|
#include "timers.h"
|
2019-07-01 18:33:28 +08:00
|
|
|
#include "config.h"
|
|
|
|
#include "overTime.h"
|
2019-07-01 19:13:13 +08:00
|
|
|
#include "database/common.h"
|
2019-06-30 00:02:14 +08:00
|
|
|
#include "log.h"
|
2019-07-01 18:51:41 +08:00
|
|
|
// global variable killed
|
|
|
|
#include "signals.h"
|
2019-07-07 16:09:18 +08:00
|
|
|
// data getter functions
|
|
|
|
#include "datastructure.h"
|
2021-09-20 17:33:13 +08:00
|
|
|
// logg_rate_limit_message()
|
|
|
|
#include "database/message-table.h"
|
2021-11-29 20:29:00 +08:00
|
|
|
// get_nprocs()
|
|
|
|
#include <sys/sysinfo.h>
|
|
|
|
// get_filepath_usage()
|
|
|
|
#include "files.h"
|
|
|
|
|
|
|
|
// Resource checking interval
|
|
|
|
// default: 300 seconds
|
|
|
|
#define RCinterval 300
|
2018-10-05 13:18:40 +08:00
|
|
|
|
2018-02-23 03:17:37 +08:00
|
|
|
bool doGC = false;
|
2017-03-15 01:46:17 +08:00
|
|
|
|
2021-09-19 02:33:42 +08:00
|
|
|
// Subtract rate-limitation count from individual client counters
|
|
|
|
// As long as client->rate_limit is still larger than the allowed
|
|
|
|
// maximum count, the rate-limitation will just continue
|
2021-02-03 23:17:21 +08:00
|
|
|
static void reset_rate_limiting(void)
|
|
|
|
{
|
|
|
|
for(int clientID = 0; clientID < counters->clients; clientID++)
|
|
|
|
{
|
|
|
|
clientsData *client = getClient(clientID, true);
|
2021-09-21 01:53:48 +08:00
|
|
|
if(!client)
|
2021-09-20 17:33:13 +08:00
|
|
|
continue;
|
|
|
|
|
2021-09-21 01:53:48 +08:00
|
|
|
// Check if we are currently rate-limiting this client
|
|
|
|
if(client->flags.rate_limited)
|
2021-09-20 17:33:13 +08:00
|
|
|
{
|
2021-09-21 01:53:48 +08:00
|
|
|
const char *clientIP = getstr(client->ippos);
|
|
|
|
|
|
|
|
// Check if we want to continue rate limiting
|
|
|
|
if(client->rate_limit > config.rate_limit.count)
|
|
|
|
{
|
|
|
|
logg("Still rate-limiting %s as it made additional %d queries", clientIP, client->rate_limit);
|
|
|
|
}
|
|
|
|
// or if rate-limiting ends for this client now
|
|
|
|
else
|
|
|
|
{
|
|
|
|
logg("Ending rate-limitation of %s", clientIP);
|
|
|
|
client->flags.rate_limited = false;
|
|
|
|
}
|
2021-09-20 18:08:58 +08:00
|
|
|
}
|
2021-09-21 01:53:48 +08:00
|
|
|
|
|
|
|
// Reset counter
|
|
|
|
client->rate_limit = 0;
|
2021-02-03 23:17:21 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-08-04 02:39:48 +08:00
|
|
|
static time_t lastRateLimitCleaner = 0;
|
2021-09-19 02:33:42 +08:00
|
|
|
// Returns how many more seconds until the current rate-limiting interval is over
|
2021-09-20 17:33:13 +08:00
|
|
|
time_t get_rate_limit_turnaround(const unsigned int rate_limit_count)
|
2021-08-04 02:39:48 +08:00
|
|
|
{
|
2021-09-20 17:33:13 +08:00
|
|
|
const unsigned int how_often = rate_limit_count/config.rate_limit.count;
|
|
|
|
return (time_t)config.rate_limit.interval*how_often - (time(NULL) - lastRateLimitCleaner);
|
2021-08-04 02:39:48 +08:00
|
|
|
}
|
|
|
|
|
2022-09-17 23:08:45 +08:00
|
|
|
static int check_space(const char *file, int LastUsage)
|
2021-11-29 20:29:00 +08:00
|
|
|
{
|
2021-11-29 22:11:06 +08:00
|
|
|
if(config.check.disk == 0)
|
2022-09-17 23:08:45 +08:00
|
|
|
return 0;
|
2021-11-29 22:11:06 +08:00
|
|
|
|
2021-11-29 20:29:00 +08:00
|
|
|
int perc = 0;
|
|
|
|
char buffer[64] = { 0 };
|
|
|
|
// Warn if space usage at the device holding the corresponding file
|
2022-09-17 23:08:45 +08:00
|
|
|
// exceeds the configured threshold and current usage is higher than
|
|
|
|
// usage in the last run (to prevent log spam)
|
|
|
|
perc = get_filepath_usage(file, buffer);
|
2023-05-08 04:11:35 +08:00
|
|
|
if(perc > config.check.disk && perc > LastUsage && perc <= 100.0)
|
2021-11-29 20:29:00 +08:00
|
|
|
log_resource_shortage(-1.0, 0, -1, perc, file, buffer);
|
2023-05-08 04:11:35 +08:00
|
|
|
|
2022-09-17 23:08:45 +08:00
|
|
|
return perc;
|
2021-11-29 20:29:00 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static void check_load(void)
|
|
|
|
{
|
2021-11-29 22:11:06 +08:00
|
|
|
if(!config.check.load)
|
|
|
|
return;
|
|
|
|
|
2021-11-29 20:29:00 +08:00
|
|
|
// Get CPU load averages
|
|
|
|
double load[3];
|
|
|
|
if (getloadavg(load, 3) == -1)
|
|
|
|
return;
|
|
|
|
|
|
|
|
// Get number of CPU cores
|
|
|
|
const int nprocs = get_nprocs();
|
|
|
|
|
|
|
|
// Warn if 15 minute average of load exceeds number of available
|
|
|
|
// processors
|
|
|
|
if(load[2] > nprocs)
|
|
|
|
log_resource_shortage(load[2], nprocs, -1, -1, NULL, NULL);
|
|
|
|
}
|
|
|
|
|
2017-03-15 01:46:17 +08:00
|
|
|
void *GC_thread(void *val)
|
|
|
|
{
|
|
|
|
// Set thread name
|
2021-04-18 17:09:52 +08:00
|
|
|
thread_names[GC] = "housekeeper";
|
|
|
|
prctl(PR_SET_NAME, thread_names[GC], 0, 0, 0);
|
2017-03-15 01:46:17 +08:00
|
|
|
|
2021-02-03 23:17:21 +08:00
|
|
|
// Remember when we last ran the actions
|
|
|
|
time_t lastGCrun = time(NULL) - time(NULL)%GCinterval;
|
2021-08-04 02:39:48 +08:00
|
|
|
lastRateLimitCleaner = time(NULL);
|
2021-11-29 20:29:00 +08:00
|
|
|
time_t lastResourceCheck = 0;
|
2021-02-18 03:53:34 +08:00
|
|
|
|
2022-09-17 23:08:45 +08:00
|
|
|
// Remember disk usage
|
|
|
|
int LastLogStorageUsage = 0;
|
|
|
|
int LastDBStorageUsage = 0;
|
|
|
|
|
2021-02-18 03:53:34 +08:00
|
|
|
// Run as long as this thread is not canceled
|
2021-04-18 17:09:52 +08:00
|
|
|
while(!killed)
|
2017-03-15 01:46:17 +08:00
|
|
|
{
|
2021-02-03 23:17:21 +08:00
|
|
|
const time_t now = time(NULL);
|
|
|
|
if((unsigned int)(now - lastRateLimitCleaner) >= config.rate_limit.interval)
|
|
|
|
{
|
|
|
|
lastRateLimitCleaner = now;
|
|
|
|
lock_shm();
|
|
|
|
reset_rate_limiting();
|
|
|
|
unlock_shm();
|
|
|
|
}
|
2021-04-18 17:09:52 +08:00
|
|
|
|
|
|
|
// Intermediate cancellation-point
|
|
|
|
if(killed)
|
|
|
|
break;
|
|
|
|
|
2021-11-29 20:29:00 +08:00
|
|
|
// Check available resources
|
|
|
|
if(now - lastResourceCheck >= RCinterval)
|
|
|
|
{
|
|
|
|
check_load();
|
2022-09-18 14:58:33 +08:00
|
|
|
LastDBStorageUsage = check_space(FTLfiles.FTL_db, LastDBStorageUsage);
|
|
|
|
LastLogStorageUsage = check_space(FTLfiles.log, LastLogStorageUsage);
|
2021-11-29 20:29:00 +08:00
|
|
|
lastResourceCheck = now;
|
|
|
|
}
|
|
|
|
|
2021-02-03 23:17:21 +08:00
|
|
|
if(now - GCdelay - lastGCrun >= GCinterval || doGC)
|
2017-03-15 01:46:17 +08:00
|
|
|
{
|
2018-02-23 03:17:37 +08:00
|
|
|
doGC = false;
|
|
|
|
// Update lastGCrun timer
|
2021-02-03 23:17:21 +08:00
|
|
|
lastGCrun = now - GCdelay - (now - GCdelay)%GCinterval;
|
2018-02-23 03:17:37 +08:00
|
|
|
|
|
|
|
// Lock FTL's data structure, since it is likely that it will be changed here
|
|
|
|
// Requests should not be processed/answered when data is about to change
|
2018-10-12 03:59:52 +08:00
|
|
|
lock_shm();
|
2018-02-23 03:17:37 +08:00
|
|
|
|
2021-04-12 17:30:07 +08:00
|
|
|
// Get minimum timestamp to keep (this can be set with MAXLOGAGE)
|
|
|
|
time_t mintime = (now - GCdelay) - config.maxlogage;
|
2019-02-15 10:08:18 +08:00
|
|
|
|
2022-03-06 23:34:00 +08:00
|
|
|
// Align the start time of this GC run to the GCinterval. This will also align with the
|
|
|
|
// oldest overTime interval after GC is done.
|
|
|
|
mintime -= mintime % GCinterval;
|
2018-02-23 03:17:37 +08:00
|
|
|
|
2019-04-23 17:18:42 +08:00
|
|
|
if(config.debug & DEBUG_GC)
|
|
|
|
{
|
|
|
|
timer_start(GC_TIMER);
|
|
|
|
char timestring[84] = "";
|
2021-02-05 15:16:15 +08:00
|
|
|
get_timestr(timestring, mintime, false);
|
2020-08-13 04:03:24 +08:00
|
|
|
logg("GC starting, mintime: %s (%llu)", timestring, (long long)mintime);
|
2019-04-23 17:18:42 +08:00
|
|
|
}
|
2018-02-23 03:17:37 +08:00
|
|
|
|
|
|
|
// Process all queries
|
2019-04-23 17:18:42 +08:00
|
|
|
int removed = 0;
|
2019-04-16 03:29:50 +08:00
|
|
|
for(long int i=0; i < counters->queries; i++)
|
2017-03-15 01:46:17 +08:00
|
|
|
{
|
2019-03-07 01:53:33 +08:00
|
|
|
queriesData* query = getQuery(i, true);
|
2019-09-06 03:04:51 +08:00
|
|
|
if(query == NULL)
|
|
|
|
continue;
|
|
|
|
|
2018-02-23 03:17:37 +08:00
|
|
|
// Test if this query is too new
|
2019-03-06 23:26:54 +08:00
|
|
|
if(query->timestamp > mintime)
|
2017-04-13 12:55:55 +08:00
|
|
|
break;
|
2018-02-23 03:17:37 +08:00
|
|
|
|
2021-01-19 22:48:25 +08:00
|
|
|
// Adjust client counter (total and overTime)
|
2019-03-07 01:53:33 +08:00
|
|
|
clientsData* client = getClient(query->clientID, true);
|
2021-07-02 23:39:58 +08:00
|
|
|
const int timeidx = getOverTimeID(query->timestamp);
|
2019-02-11 05:55:10 +08:00
|
|
|
overTime[timeidx].total--;
|
2019-09-06 03:04:51 +08:00
|
|
|
if(client != NULL)
|
2021-01-19 22:48:25 +08:00
|
|
|
change_clientcount(client, -1, 0, timeidx, -1);
|
2018-02-23 03:17:37 +08:00
|
|
|
|
2018-05-10 20:22:58 +08:00
|
|
|
// Adjust domain counter (no overTime information)
|
2019-03-07 01:53:33 +08:00
|
|
|
domainsData* domain = getDomain(query->domainID, true);
|
2019-09-06 03:04:51 +08:00
|
|
|
if(domain != NULL)
|
|
|
|
domain->count--;
|
2018-04-09 04:51:19 +08:00
|
|
|
|
2020-02-11 16:53:22 +08:00
|
|
|
// Get upstream pointer
|
2019-03-07 00:30:44 +08:00
|
|
|
|
2018-02-23 03:17:37 +08:00
|
|
|
// Change other counters according to status of this query
|
2019-03-06 23:26:54 +08:00
|
|
|
switch(query->status)
|
2018-02-23 03:17:37 +08:00
|
|
|
{
|
|
|
|
case QUERY_UNKNOWN:
|
|
|
|
// Unknown (?)
|
|
|
|
break;
|
2020-10-08 22:50:40 +08:00
|
|
|
case QUERY_FORWARDED: // (fall through)
|
2020-10-19 00:55:39 +08:00
|
|
|
case QUERY_RETRIED: // (fall through)
|
|
|
|
case QUERY_RETRIED_DNSSEC:
|
2018-02-23 03:17:37 +08:00
|
|
|
// Forwarded to an upstream DNS server
|
2021-08-22 17:56:59 +08:00
|
|
|
// Adjusting counters is done below in moveOverTimeMemory()
|
2018-02-23 03:17:37 +08:00
|
|
|
break;
|
|
|
|
case QUERY_CACHE:
|
2022-11-07 20:56:44 +08:00
|
|
|
case QUERY_CACHE_STALE:
|
2018-02-23 03:17:37 +08:00
|
|
|
// Answered from local cache _or_ local config
|
|
|
|
break;
|
2018-11-05 05:24:49 +08:00
|
|
|
case QUERY_GRAVITY: // Blocked by Pi-hole's blocking lists (fall through)
|
|
|
|
case QUERY_BLACKLIST: // Exact blocked (fall through)
|
2020-01-29 16:33:09 +08:00
|
|
|
case QUERY_REGEX: // Regex blocked (fall through)
|
2019-02-05 00:40:46 +08:00
|
|
|
case QUERY_EXTERNAL_BLOCKED_IP: // Blocked by upstream provider (fall through)
|
|
|
|
case QUERY_EXTERNAL_BLOCKED_NXRA: // Blocked by upstream provider (fall through)
|
|
|
|
case QUERY_EXTERNAL_BLOCKED_NULL: // Blocked by upstream provider (fall through)
|
2020-01-29 16:33:09 +08:00
|
|
|
case QUERY_GRAVITY_CNAME: // Gravity domain in CNAME chain (fall through)
|
|
|
|
case QUERY_REGEX_CNAME: // Regex blacklisted domain in CNAME chain (fall through)
|
2021-08-18 16:57:17 +08:00
|
|
|
case QUERY_BLACKLIST_CNAME: // Exactly blacklisted domain in CNAME chain (fall through)
|
|
|
|
case QUERY_DBBUSY: // Blocked because gravity database was busy
|
2022-04-24 04:08:42 +08:00
|
|
|
case QUERY_SPECIAL_DOMAIN: // Blocked by special domain handling
|
2019-09-06 03:04:51 +08:00
|
|
|
if(domain != NULL)
|
|
|
|
domain->blockedcount--;
|
|
|
|
if(client != NULL)
|
2020-08-11 06:55:16 +08:00
|
|
|
change_clientcount(client, 0, -1, -1, 0);
|
2018-02-23 03:17:37 +08:00
|
|
|
break;
|
2021-04-07 07:59:30 +08:00
|
|
|
case QUERY_IN_PROGRESS: // Don't have to do anything here
|
2020-07-09 05:15:55 +08:00
|
|
|
case QUERY_STATUS_MAX: // fall through
|
2018-02-23 03:17:37 +08:00
|
|
|
default:
|
|
|
|
/* That cannot happen */
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Update reply counters
|
2021-04-07 02:01:47 +08:00
|
|
|
counters->reply[query->reply]--;
|
2017-03-15 01:46:17 +08:00
|
|
|
|
2018-02-23 03:17:37 +08:00
|
|
|
// Update type counters
|
2019-03-06 23:26:54 +08:00
|
|
|
if(query->type >= TYPE_A && query->type < TYPE_MAX)
|
2018-02-23 03:17:37 +08:00
|
|
|
{
|
2019-03-06 23:26:54 +08:00
|
|
|
counters->querytype[query->type-1]--;
|
2018-02-23 03:17:37 +08:00
|
|
|
}
|
2017-03-15 01:46:17 +08:00
|
|
|
|
2021-04-07 02:01:47 +08:00
|
|
|
// Set query again to UNKNOWN to reset the counters
|
|
|
|
query_set_status(query, QUERY_UNKNOWN);
|
|
|
|
|
2021-04-07 07:45:53 +08:00
|
|
|
// Finally, remove the last trace of this query
|
|
|
|
counters->status[QUERY_UNKNOWN]--;
|
|
|
|
|
2018-02-23 03:17:37 +08:00
|
|
|
// Count removed queries
|
|
|
|
removed++;
|
2017-03-15 01:46:17 +08:00
|
|
|
}
|
|
|
|
|
2019-03-15 04:00:38 +08:00
|
|
|
// Only perform memory operations when we actually removed queries
|
|
|
|
if(removed > 0)
|
|
|
|
{
|
|
|
|
// Move memory forward to keep only what we want
|
|
|
|
// Note: for overlapping memory blocks, memmove() is a safer approach than memcpy()
|
|
|
|
// Example: (I = now invalid, X = still valid queries, F = free space)
|
|
|
|
// Before: IIIIIIXXXXFF
|
|
|
|
// After: XXXXFFFFFFFF
|
2021-09-14 14:31:14 +08:00
|
|
|
queriesData *dest = getQuery(0, true);
|
|
|
|
queriesData *src = getQuery(removed, true);
|
|
|
|
if(dest && src)
|
|
|
|
memmove(dest, src, (counters->queries - removed)*sizeof(queriesData));
|
2019-03-15 04:00:38 +08:00
|
|
|
|
|
|
|
// Update queries counter
|
|
|
|
counters->queries -= removed;
|
|
|
|
// Update DB index as total number of queries reduced
|
|
|
|
lastdbindex -= removed;
|
|
|
|
|
|
|
|
// ensure remaining memory is zeroed out (marked as "F" in the above example)
|
2021-09-14 14:31:14 +08:00
|
|
|
queriesData *tail = getQuery(counters->queries, true);
|
|
|
|
if(tail)
|
|
|
|
memset(tail, 0, (counters->queries_MAX - counters->queries)*sizeof(queriesData));
|
2019-03-15 04:00:38 +08:00
|
|
|
}
|
2017-09-06 22:59:35 +08:00
|
|
|
|
2019-02-10 19:54:18 +08:00
|
|
|
// Determine if overTime memory needs to get moved
|
2019-02-15 10:08:18 +08:00
|
|
|
moveOverTimeMemory(mintime);
|
2019-02-10 19:54:18 +08:00
|
|
|
|
2019-04-16 03:29:50 +08:00
|
|
|
if(config.debug & DEBUG_GC)
|
|
|
|
logg("Notice: GC removed %i queries (took %.2f ms)", removed, timer_elapsed_msec(GC_TIMER));
|
2017-03-15 01:46:17 +08:00
|
|
|
|
2018-02-23 03:17:37 +08:00
|
|
|
// Release thread lock
|
2018-10-12 03:59:52 +08:00
|
|
|
unlock_shm();
|
2018-04-12 00:16:41 +08:00
|
|
|
|
2018-07-31 17:08:37 +08:00
|
|
|
// After storing data in the database for the next time,
|
|
|
|
// we should scan for old entries, which will then be deleted
|
|
|
|
// to free up pages in the database and prevent it from growing
|
|
|
|
// ever larger and larger
|
|
|
|
DBdeleteoldqueries = true;
|
2018-02-23 03:17:37 +08:00
|
|
|
}
|
2021-04-18 17:09:52 +08:00
|
|
|
thread_sleepms(GC, 1000);
|
2018-02-23 03:17:37 +08:00
|
|
|
}
|
2017-03-15 01:46:17 +08:00
|
|
|
|
2021-04-18 17:09:52 +08:00
|
|
|
logg("Terminating GC thread");
|
2017-03-15 01:46:17 +08:00
|
|
|
return NULL;
|
|
|
|
}
|