Github
/
FTL
mirror of
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'development-v6' into new/validator 

Signed-off-by: DL6ER <dl6er@dl6er.de>
This commit is contained in:
DL6ER 2023-11-26 21:48:13 +01:00
parent 98b5a92b52 30c3e9cbb0
commit f7b47e6b51
No known key found for this signature in database
GPG Key ID: 00135ACBD90B28DD
27 changed files with 2426 additions and 1401 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch
View File

@ -25,6 +25,6 @@ index 6280ebf6..a5e82f70 100644
char *zHistory;
int nHistory;
+ print_FTL_version();
#if SHELL_WIN_UTF8_OPT
switch( console_utf8_in+2*console_utf8_out ){
default: case 0: break;
#if CIO_WIN_WC_XLATE
# define SHELL_CIO_CHAR_SET (stdout_is_console? " (UTF-16 console I/O)" : "")
#else

4
src/api/docs/content/specs/logs.yaml
View File

@ -132,6 +132,10 @@ components:
type: integer
description: Next ID to query if checking for new log lines
example: 229
pid:
type: integer
description: Process ID of FTL. When this changes, FTL was restarted and nextID should be reset to 0.
example: 2258
file:
type: string
description: Path to respective log file on disk

4
src/api/list.c
View File

@ -58,9 +58,11 @@ static int api_list_read(struct ftl_conn *api,
char *name = NULL;
if(table.client != NULL)
{
// Try to obtain hostname if this is a valid IP address
// Try to obtain hostname
if(isValidIPv4(table.client) || isValidIPv6(table.client))
name = getNameFromIP(NULL, table.client);
else if(isMAC(table.client))
name = getNameFromMAC(table.client);
}
JSON_COPY_STR_TO_OBJECT(row, "client", table.client);

3
src/api/logs.c
View File

@ -15,6 +15,8 @@
// struct fifologData
#include "log.h"
#include "config/config.h"
// main_pid()
#include "signals.h"
// fifologData is allocated in shared memory for cross-fork compatibility
int api_logs(struct ftl_conn *api)
@ -70,6 +72,7 @@ int api_logs(struct ftl_conn *api)
}
JSON_ADD_ITEM_TO_OBJECT(json, "log", log);
JSON_ADD_NUMBER_TO_OBJECT(json, "nextID", fifo_log->logs[api->opts.which].next_id);
JSON_ADD_NUMBER_TO_OBJECT(json, "pid", main_pid());
// Add file name
const char *logfile = NULL;

21
src/args.c
View File

@ -62,6 +62,8 @@
#include "config/password.h"
// idn2_to_ascii_lz()
#include <idn2.h>
// sha256sum()
#include "files.h"
// defined in dnsmasq.c
extern void print_dnsmasq_version(const char *yellow, const char *green, const char *bold, const char *normal);
@ -476,6 +478,24 @@ void parse_args(int argc, char* argv[])
}
}
// sha256sum mode
if(argc == 3 && strcmp(argv[1], "sha256sum") == 0)
{
// Enable stdout printing
cli_mode = true;
uint8_t checksum[SHA256_DIGEST_SIZE];
if(!sha256sum(argv[2], checksum))
exit(EXIT_FAILURE);
// Convert checksum to hex string
char hex[SHA256_DIGEST_SIZE*2+1];
sha256_raw_to_hex(checksum, hex);
// Print result
printf("%s %s\n", hex, argv[2]);
exit(EXIT_SUCCESS);
}
// start from 1, as argv[0] is the executable name
for(int i = 1; i < argc; i++)
{
@ -934,6 +954,7 @@ void parse_args(int argc, char* argv[])
printf(" Decoding: %spihole-FTL idn2 -d %spunycode%s\n\n", green, cyan, normal);
printf("%sOther:%s\n", yellow, normal);
printf("\t%ssha256sum %sfile%s Calculate SHA256 checksum of a file\n", green, cyan, normal);
printf("\t%sdhcp-discover%s Discover DHCP servers in the local\n", green, normal);
printf("\t network\n");
printf("\t%sarp-scan %s[-a/-x]%s Use ARP to scan local network for\n", green, cyan, normal);

61
src/config/config.c
View File

@ -31,9 +31,12 @@
#include "signals.h"
// validation functions
#include "config/validator.h"
// sha256sum()
#include "files.h"
struct config config = { 0 };
static bool config_initialized = false;
uint8_t last_checksum[SHA256_DIGEST_SIZE] = { 0 };
// Private prototypes
static bool port_in_use(const in_port_t port);
@ -1388,28 +1391,37 @@ void reset_config(struct conf_item *conf_item)
}
}
void readFTLconf(struct config *conf, const bool rewrite)
bool readFTLconf(struct config *conf, const bool rewrite)
{
// Initialize config with default values
initConfig(conf);
// First try to read TOML config file
if(readFTLtoml(NULL, conf, NULL, rewrite, NULL))
// If we cannot parse /etc/pihole.toml (due to missing or invalid syntax),
// we try to read the rotated files in /etc/pihole/config_backup starting at
// the most recent one and going back in time until we find a valid config
for(unsigned int i = 0; i < MAX_ROTATIONS; i++)
{
// If successful, we write the config file back to disk
// to ensure that all options are present and comments
// about options deviating from the default are present
if(rewrite)
if(readFTLtoml(NULL, conf, NULL, rewrite, NULL, i))
{
writeFTLtoml(true);
write_dnsmasq_config(conf, false, NULL);
write_custom_list();
// If successful, we write the config file back to disk
// to ensure that all options are present and comments
// about options deviating from the default are present
if(rewrite)
{
writeFTLtoml(true);
write_dnsmasq_config(conf, false, NULL);
write_custom_list();
}
return true;
}
return;
}
// On error, try to read legacy (pre-v6.0) config file. If successful,
// we move the legacy config file out of our way
log_info("No config file nor backup available, using defaults");
// If no previous config file could be read, we are likely either running
// for the first time or we are upgrading from a version prior to v6.0
// In this case, we try to read the legacy config files
const char *path = "";
if((path = readFTLlegacy(conf)) != NULL)
{
@ -1449,7 +1461,7 @@ void readFTLconf(struct config *conf, const bool rewrite)
if(ports == NULL)
{
log_err("Unable to allocate memory for default ports string");
return;
return false;
}
// Create the string
snprintf(ports, 32, "%d,%ds", http_port, https_port);
@ -1473,6 +1485,8 @@ void readFTLconf(struct config *conf, const bool rewrite)
writeFTLtoml(true);
write_dnsmasq_config(conf, false, NULL);
write_custom_list();
return false;
}
bool getLogFilePath(void)
@ -1581,12 +1595,29 @@ void replace_config(struct config *newconf)
void reread_config(void)
{
// Create checksum of config file
uint8_t checksum[SHA256_DIGEST_SIZE];
if(!sha256sum(GLOBALTOMLPATH, checksum))
{
log_err("Unable to create checksum of %s, not re-reading config file", GLOBALTOMLPATH);
return;
}
// Compare checksums
if(memcmp(checksum, last_checksum, SHA256_DIGEST_SIZE) == 0)
{
log_debug(DEBUG_CONFIG, "Checksum of %s has not changed, not re-reading config file", GLOBALTOMLPATH);
return;
}
log_info("Reloading config due to pihole.toml change");
struct config conf_copy;
duplicate_config(&conf_copy, &config);
// Read TOML config file
bool restart = false;
if(readFTLtoml(&config, &conf_copy, NULL, true, &restart))
if(readFTLtoml(&config, &conf_copy, NULL, true, &restart, 0))
{
// Install new configuration
log_debug(DEBUG_CONFIG, "Loaded configuration is valid, installing it");
@ -1614,7 +1645,7 @@ void reread_config(void)
else
{
// New configuration is invalid, restore old one
log_debug(DEBUG_CONFIG, "Loaded configuration is invalid, restoring old one");
log_debug(DEBUG_CONFIG, "Modified config file is invalid, discarding and overwriting with current configuration");
free_config(&conf_copy);
}

2
src/config/config.h
View File

@ -328,7 +328,7 @@ void set_debug_flags(struct config *conf);
void set_all_debug(struct config *conf, const bool status);
void initConfig(struct config *conf);
void reset_config(struct conf_item *conf_item);
void readFTLconf(struct config *conf, const bool rewrite);
bool readFTLconf(struct config *conf, const bool rewrite);
bool getLogFilePath(void);
struct conf_item *get_conf_item(struct config *conf, const unsigned int n);
struct conf_item *get_debug_item(struct config *conf, const enum debug_flag debug);

24
src/config/inotify.c
View File

@ -106,18 +106,38 @@ bool check_inotify_event(void)
// Check if this is the event we are looking for
if(event->mask & IN_CLOSE_WRITE)
{
// File opened for writing was closed
log_debug(DEBUG_INOTIFY, "File written: "WATCHDIR"/%s", event->name);
if(strcmp(event->name, "pihole.toml") == 0)
config_changed = true;
}
else if(event->mask & IN_CREATE)
{
// File was created
log_debug(DEBUG_INOTIFY, "File created: "WATCHDIR"/%s", event->name);
else if(event->mask & IN_MOVE)
log_debug(DEBUG_INOTIFY, "File moved: "WATCHDIR"/%s", event->name);
}
else if(event->mask & IN_MOVED_FROM)
{
// File was moved (source)
log_debug(DEBUG_INOTIFY, "File moved from: "WATCHDIR"/%s", event->name);
}
else if(event->mask & IN_MOVED_TO)
{
// File was moved (target)
log_debug(DEBUG_INOTIFY, "File moved to: "WATCHDIR"/%s", event->name);
if(strcmp(event->name, "pihole.toml") == 0)
config_changed = true;
}
else if(event->mask & IN_DELETE)
{
// File was deleted
log_debug(DEBUG_INOTIFY, "File deleted: "WATCHDIR"/%s", event->name);
}
else if(event->mask & IN_IGNORED)
{
// Watch descriptor was removed
log_warn("Inotify watch descriptor for "WATCHDIR" was removed (directory deleted or unmounted?)");
}
else
log_debug(DEBUG_INOTIFY, "Unknown event (%X) on watched file: "WATCHDIR"/%s", event->mask, event->name);
}

45
src/config/toml_helper.c
View File

@ -20,27 +20,64 @@
#include "files.h"
//set_and_check_password()
#include "config/password.h"
// PATH_MAX
#include <limits.h>
// Open the TOML file for reading or writing
FILE * __attribute((malloc)) __attribute((nonnull(1,2))) openFTLtoml(const char *path, const char *mode)
FILE * __attribute((malloc)) __attribute((nonnull(1))) openFTLtoml(const char *mode, const unsigned int version)
{
// Try to open file in requested mode
FILE *fp = fopen(path, mode);
// This should not happen, install a safeguard anyway to unveil
// possible future coding issues early on
if(mode[0] == 'w' && version != 0)
{
log_crit("Writing to version != 0 is not supported in openFTLtoml(%s,%u)",
mode, version);
exit(EXIT_FAILURE);
}
// Build filename based on version
char filename[PATH_MAX] = { 0 };
if(version == 0)
{
// Use global config file
strncpy(filename, GLOBALTOMLPATH, sizeof(filename));
// Append ".tmp" if we are writing
if(mode[0] == 'w')
strncat(filename, ".tmp", sizeof(filename));
}
else
{
// Use rotated config file
snprintf(filename, sizeof(filename), BACKUP_DIR"/pihole.toml.%u", version);
}
// Try to open config file
FILE *fp = fopen(filename, mode);
// Return early if opening failed
if(!fp)
{
log_info("Config %sfile %s not available: %s",
version > 0 ? "backup " : "", filename, strerror(errno));
return NULL;
}
// Lock file, may block if the file is currently opened
if(flock(fileno(fp), LOCK_EX) != 0)
{
const int _e = errno;
log_err("Cannot open FTL's config file in exclusive mode: %s", strerror(errno));
log_err("Cannot open config file %s in exclusive mode: %s",
filename, strerror(errno));
fclose(fp);
errno = _e;
return NULL;
}
// Log if we are using a backup file
if(version > 0)
log_info("Using config backup %s", filename);
errno = 0;
return fp;
}

2
src/config/toml_helper.h
View File

@ -17,7 +17,7 @@
#include "tomlc99/toml.h"
void indentTOML(FILE *fp, const unsigned int indent);
FILE *openFTLtoml(const char *path, const char *mode) __attribute((malloc)) __attribute((nonnull(1,2)));
FILE *openFTLtoml(const char *mode, const unsigned int version) __attribute((malloc)) __attribute((nonnull(1)));
void closeFTLtoml(FILE *fp);
void print_comment(FILE *fp, const char *str, const char *intro, const unsigned int width, const unsigned int indent);
void print_toml_allowed_values(cJSON *allowed_values, FILE *fp, const unsigned int width, const unsigned int indent);

21
src/config/toml_reader.c
View File

@ -25,18 +25,19 @@
#include "api/api.h"
// Private prototypes
static toml_table_t *parseTOML(void);
static toml_table_t *parseTOML(const unsigned int version);
static void reportDebugFlags(void);
bool readFTLtoml(struct config *oldconf, struct config *newconf,
toml_table_t *toml, const bool verbose, bool *restart)
toml_table_t *toml, const bool verbose, bool *restart,
const unsigned int version)
{
// Parse lines in the config file if we did not receive a pointer to a TOML
// table from an imported Teleporter file
bool teleporter = (toml != NULL);
if(!teleporter)
{
toml = parseTOML();
toml = parseTOML(version);
if(!toml)
return false;
}
@ -59,8 +60,8 @@ bool readFTLtoml(struct config *oldconf, struct config *newconf,
}
set_debug_flags(newconf);
log_debug(DEBUG_CONFIG, "Reading %s TOML config file: full config",
teleporter ? "teleporter" : "default");
log_debug(DEBUG_CONFIG, "Reading %s TOML config file",
teleporter ? "teleporter" : version == 0 ? "default" : "backup");
// Read all known config items
for(unsigned int i = 0; i < CONFIG_ELEMENTS; i++)
@ -134,16 +135,12 @@ bool readFTLtoml(struct config *oldconf, struct config *newconf,
}
// Parse TOML config file
static toml_table_t *parseTOML(void)
static toml_table_t *parseTOML(const unsigned int version)
{
// Try to open default config file. Use fallback if not found
FILE *fp;
if((fp = openFTLtoml(GLOBALTOMLPATH, "r")) == NULL)
{
log_warn("No config file available (%s), using defaults",
strerror(errno));
if((fp = openFTLtoml("r", version)) == NULL)
return NULL;
}
// Parse lines in the config file
char errbuf[200];
@ -167,7 +164,7 @@ bool getLogFilePathTOML(void)
{
log_debug(DEBUG_CONFIG, "Reading TOML config file: log file path");
toml_table_t *conf = parseTOML();
toml_table_t *conf = parseTOML(0);
if(!conf)
return false;

3
src/config/toml_reader.h
View File

@ -14,7 +14,8 @@
#include "tomlc99/toml.h"
bool readFTLtoml(struct config *oldconf, struct config *newconf,
toml_table_t *toml, const bool verbose, bool *restart);
toml_table_t *toml, const bool verbose, bool *restart,
const unsigned int version);
bool getLogFilePathTOML(void);
#endif //TOML_READER_H

8
src/config/toml_writer.c
View File

@ -22,6 +22,9 @@
// files_different()
#include "files.h"
// defined in config/config.c
extern uint8_t last_checksum[SHA256_DIGEST_SIZE];
static void migrate_config(void)
{
// Migrating dhcp.domain -> dns.domain
@ -45,7 +48,7 @@ bool writeFTLtoml(const bool verbose)
{
// Try to open a temporary config file for writing
FILE *fp;
if((fp = openFTLtoml(GLOBALTOMLPATH".tmp", "w")) == NULL)
if((fp = openFTLtoml("w", 0)) == NULL)
{
log_warn("Cannot write to FTL config file (%s), content not updated", strerror(errno));
return false;
@ -176,5 +179,8 @@ bool writeFTLtoml(const bool verbose)
log_debug(DEBUG_CONFIG, "pihole.toml unchanged");
}
if(!sha256sum(GLOBALTOMLPATH, last_checksum))
log_err("Unable to create checksum of %s", GLOBALTOMLPATH);
return true;
}

104
src/database/network-table.c
View File

@ -2144,6 +2144,84 @@ char *__attribute__((malloc)) getNameFromIP(sqlite3 *db, const char *ipaddr)
return name;
}
// Get most recently seen host name of device identified by MAC address
char *__attribute__((malloc)) getNameFromMAC(const char *client)
{
// Return early if database is known to be broken
if(FTLDBerror())
return NULL;
log_debug(DEBUG_DATABASE,"Looking up host name for %s", client);
// Open pihole-FTL.db database file
sqlite3 *db = NULL;
if((db = dbopen(false, false)) == NULL)
{
log_warn("getNameFromMAC(\"%s\") - Failed to open DB", client);
return NULL;
}
// Check for a host name associated with the given client as MAC address
// COLLATE NOCASE: Case-insensitive comparison
const char *querystr = "SELECT name FROM network_addresses "
"WHERE name IS NOT NULL AND "
"network_id = (SELECT id FROM network WHERE hwaddr = ? COLLATE NOCASE) "
"ORDER BY lastSeen DESC LIMIT 1";
sqlite3_stmt *stmt = NULL;
int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL);
if(rc != SQLITE_OK)
{
log_err("getNameFromMAC(\"%s\") - SQL error prepare: %s",
client, sqlite3_errstr(rc));
dbclose(&db);
return NULL;
}
// Bind client to prepared statement
if((rc = sqlite3_bind_text(stmt, 1, client, -1, SQLITE_STATIC)) != SQLITE_OK)
{
log_warn("getNameFromMAC(\"%s\"): Failed to bind ip: %s",
client, sqlite3_errstr(rc));
checkFTLDBrc(rc);
sqlite3_reset(stmt);
sqlite3_finalize(stmt);
dbclose(&db);
return NULL;
}
char *name = NULL;
rc = sqlite3_step(stmt);
if(rc == SQLITE_ROW)
{
// Database record found (result might be empty)
name = strdup((char*)sqlite3_column_text(stmt, 0));
if(config.debug.resolver.v.b)
log_debug(DEBUG_RESOLVER, "Found database host name (by MAC) %s -> %s",
client, name);
}
else if(rc == SQLITE_DONE)
{
// Not found
if(config.debug.resolver.v.b)
log_debug(DEBUG_RESOLVER, " ---> not found");
}
else
{
// Error
checkFTLDBrc(rc);
return NULL;
}
// Finalize statement and close database handle
sqlite3_reset(stmt);
sqlite3_finalize(stmt);
dbclose(&db);
return name;
}
// Get interface of device identified by IP address
char *__attribute__((malloc)) getIfaceFromIP(sqlite3 *db, const char *ipaddr)
{
@ -2425,3 +2503,29 @@ bool networkTable_deleteDevice(sqlite3 *db, const int id, const char **message)
return true;
}
// Counting number of occurrences of a specific char in a string
static size_t __attribute__ ((pure)) count_char(const char *haystack, const char needle)
{
size_t count = 0u;
while(*haystack)
if (*haystack++ == needle)
++count;
return count;
}
// Identify MAC addresses using a set of suitable criteria
bool __attribute__ ((pure)) isMAC(const char *input)
{
if(input != NULL && // Valid input
strlen(input) == 17u && // MAC addresses are always 17 chars long (6 bytes + 5 colons)
count_char(input, ':') == 5u && // MAC addresses always have 5 colons
strstr(input, "::") == NULL) // No double-colons (IPv6 address abbreviation)
{
// This is a MAC address of the form AA:BB:CC:DD:EE:FF
return true;
}
// Not a MAC address
return false;
}

8
src/database/network-table.h
View File

@ -18,12 +18,14 @@ bool create_network_addresses_with_names_table(sqlite3 *db);
void parse_neighbor_cache(sqlite3 *db);
void updateMACVendorRecords(sqlite3 *db);
bool unify_hwaddr(sqlite3 *db);
char* __attribute__((malloc)) getMACfromIP(sqlite3 *db, const char* ipaddr);
char *getMACfromIP(sqlite3 *db, const char* ipaddr) __attribute__((malloc));
int getAliasclientIDfromIP(sqlite3 *db, const char *ipaddr);
char* __attribute__((malloc)) getNameFromIP(sqlite3 *db, const char* ipaddr);
char* __attribute__((malloc)) getIfaceFromIP(sqlite3 *db, const char* ipaddr);
char *getNameFromIP(sqlite3 *db, const char* ipaddr) __attribute__((malloc));
char *getNameFromMAC(const char *client) __attribute__((malloc));
char *getIfaceFromIP(sqlite3 *db, const char* ipaddr) __attribute__((malloc));
void resolveNetworkTableNames(void);
bool flush_network_table(void);
bool isMAC(const char *input) __attribute__ ((pure));
typedef struct {
unsigned int id;

4
src/database/session-table.c
View File

@ -198,8 +198,8 @@ bool backup_db_sessions(struct session *sessions, const uint16_t max_sessions)
return false;
}
log_info("Stored %u/%u API session%s in the database",
api_sessions, max_sessions, max_sessions == 1 ? "" : "s");
log_info("Stored %u API session%s in the database",
api_sessions, api_sessions == 1 ? "" : "s");
// Close database connection
dbclose(&db);

3030
src/database/shell.c
View File

File diff suppressed because it is too large Load Diff

26
src/database/sqlite3-ext.c
View File

@ -29,32 +29,6 @@
// isMAC()
#include "network-table.h"
// Counting number of occurrences of a specific char in a string
static size_t __attribute__ ((pure)) count_char(const char *haystack, const char needle)
{
size_t count = 0u;
while(*haystack)
if (*haystack++ == needle)
++count;
return count;
}
// Identify MAC addresses using a set of suitable criteria
static bool __attribute__ ((pure)) isMAC(const char *input)
{
if(input != NULL && // Valid input
strlen(input) == 17u && // MAC addresses are always 17 chars long (6 bytes + 5 colons)
count_char(input, ':') == 5u && // MAC addresses always have 5 colons
strstr(input, "::") == NULL) // No double-colons (IPv6 address abbreviation)
{
// This is a MAC address of the form AA:BB:CC:DD:EE:FF
return true;
}
// Not a MAC address
return false;
}
static void subnet_match_impl(sqlite3_context *context, int argc, sqlite3_value **argv)
{
// Exactly two arguments should be submitted to this routine

255
src/database/sqlite3.c
View File

@ -1,6 +1,6 @@
/******************************************************************************
** This file is an amalgamation of many separate C source files from SQLite
** version 3.44.0. By combining all the individual C code files into this
** version 3.44.1. By combining all the individual C code files into this
** single large file, the entire code can be compiled as a single translation
** unit. This allows many compilers to do optimizations that would not be
** possible if the files were compiled separately. Performance improvements
@ -18,7 +18,7 @@
** separate file. This file contains only code for the core SQLite library.
**
** The content in this amalgamation comes from Fossil check-in
** 17129ba1ff7f0daf37100ee82d507aef7827.
** d295f48e8f367b066b881780c98bdf980a1d.
*/
#define SQLITE_CORE 1
#define SQLITE_AMALGAMATION 1
@ -459,9 +459,9 @@ extern "C" {
** [sqlite3_libversion_number()], [sqlite3_sourceid()],
** [sqlite_version()] and [sqlite_source_id()].
*/
#define SQLITE_VERSION "3.44.0"
#define SQLITE_VERSION_NUMBER 3044000
#define SQLITE_SOURCE_ID "2023-11-01 11:23:50 17129ba1ff7f0daf37100ee82d507aef7827cf38de1866e2633096ae6ad81301"
#define SQLITE_VERSION "3.44.1"
#define SQLITE_VERSION_NUMBER 3044001
#define SQLITE_SOURCE_ID "2023-11-22 14:18:12 d295f48e8f367b066b881780c98bdf980a1d550397d5ba0b0e49842c95b3e8b4"
/*
** CAPI3REF: Run-Time Library Version Numbers
@ -5886,13 +5886,27 @@ SQLITE_API int sqlite3_create_window_function(
** </dd>
**
** [[SQLITE_SUBTYPE]] <dt>SQLITE_SUBTYPE</dt><dd>
** The SQLITE_SUBTYPE flag indicates to SQLite that a function may call
** The SQLITE_SUBTYPE flag indicates to SQLite that a function might call
** [sqlite3_value_subtype()] to inspect the sub-types of its arguments.
** Specifying this flag makes no difference for scalar or aggregate user
** functions. However, if it is not specified for a user-defined window
** function, then any sub-types belonging to arguments passed to the window
** function may be discarded before the window function is called (i.e.
** sqlite3_value_subtype() will always return 0).
** This flag instructs SQLite to omit some corner-case optimizations that
** might disrupt the operation of the [sqlite3_value_subtype()] function,
** causing it to return zero rather than the correct subtype().
** SQL functions that invokes [sqlite3_value_subtype()] should have this
** property. If the SQLITE_SUBTYPE property is omitted, then the return
** value from [sqlite3_value_subtype()] might sometimes be zero even though
** a non-zero subtype was specified by the function argument expression.
**
** [[SQLITE_RESULT_SUBTYPE]] <dt>SQLITE_RESULT_SUBTYPE</dt><dd>
** The SQLITE_RESULT_SUBTYPE flag indicates to SQLite that a function might call
** [sqlite3_result_subtype()] to cause a sub-type to be associated with its
** result.
** Every function that invokes [sqlite3_result_subtype()] should have this
** property. If it does not, then the call to [sqlite3_result_subtype()]
** might become a no-op if the function is used as term in an
** [expression index]. On the other hand, SQL functions that never invoke
** [sqlite3_result_subtype()] should avoid setting this property, as the
** purpose of this property is to disable certain optimizations that are
** incompatible with subtypes.
** </dd>
** </dl>
*/
@ -5900,6 +5914,7 @@ SQLITE_API int sqlite3_create_window_function(
#define SQLITE_DIRECTONLY 0x000080000
#define SQLITE_SUBTYPE 0x000100000
#define SQLITE_INNOCUOUS 0x000200000
#define SQLITE_RESULT_SUBTYPE 0x001000000
/*
** CAPI3REF: Deprecated Functions
@ -6096,6 +6111,12 @@ SQLITE_API int sqlite3_value_encoding(sqlite3_value*);
** information can be used to pass a limited amount of context from
** one SQL function to another. Use the [sqlite3_result_subtype()]
** routine to set the subtype for the return value of an SQL function.
**
** Every [application-defined SQL function] that invoke this interface
** should include the [SQLITE_SUBTYPE] property in the text
** encoding argument when the function is [sqlite3_create_function|registered].
** If the [SQLITE_SUBTYPE] property is omitted, then sqlite3_value_subtype()
** might return zero instead of the upstream subtype in some corner cases.
*/
SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value*);
@ -6226,14 +6247,22 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*);
** <li> ^(when sqlite3_set_auxdata() is invoked again on the same
** parameter)^, or
** <li> ^(during the original sqlite3_set_auxdata() call when a memory
** allocation error occurs.)^ </ul>
** allocation error occurs.)^
** <li> ^(during the original sqlite3_set_auxdata() call if the function
** is evaluated during query planning instead of during query execution,
** as sometimes happens with [SQLITE_ENABLE_STAT4].)^ </ul>
**
** Note the last bullet in particular. The destructor X in
** Note the last two bullets in particular. The destructor X in
** sqlite3_set_auxdata(C,N,P,X) might be called immediately, before the
** sqlite3_set_auxdata() interface even returns. Hence sqlite3_set_auxdata()
** should be called near the end of the function implementation and the
** function implementation should not make any use of P after
** sqlite3_set_auxdata() has been called.
** sqlite3_set_auxdata() has been called. Furthermore, a call to
** sqlite3_get_auxdata() that occurs immediately after a corresponding call
** to sqlite3_set_auxdata() might still return NULL if an out-of-memory
** condition occurred during the sqlite3_set_auxdata() call or if the
** function is being evaluated during query planning rather than during
** query execution.
**
** ^(In practice, auxiliary data is preserved between function calls for
** function parameters that are compile-time constants, including literal
@ -6507,6 +6536,20 @@ SQLITE_API int sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n);
** higher order bits are discarded.
** The number of subtype bytes preserved by SQLite might increase
** in future releases of SQLite.
**
** Every [application-defined SQL function] that invokes this interface
** should include the [SQLITE_RESULT_SUBTYPE] property in its
** text encoding argument when the SQL function is
** [sqlite3_create_function|registered]. If the [SQLITE_RESULT_SUBTYPE]
** property is omitted from the function that invokes sqlite3_result_subtype(),
** then in some cases the sqlite3_result_subtype() might fail to set
** the result subtype.
**
** If SQLite is compiled with -DSQLITE_STRICT_SUBTYPE=1, then any
** SQL function that invokes the sqlite3_result_subtype() interface
** and that does not have the SQLITE_RESULT_SUBTYPE property will raise
** an error. Future versions of SQLite might enable -DSQLITE_STRICT_SUBTYPE=1
** by default.
*/
SQLITE_API void sqlite3_result_subtype(sqlite3_context*,unsigned int);
@ -17811,14 +17854,15 @@ struct FuncDestructor {
#define SQLITE_FUNC_SLOCHNG 0x2000 /* "Slow Change". Value constant during a
** single query - might change over time */
#define SQLITE_FUNC_TEST 0x4000 /* Built-in testing functions */
/* 0x8000 -- available for reuse */
#define SQLITE_FUNC_RUNONLY 0x8000 /* Cannot be used by valueFromFunction */
#define SQLITE_FUNC_WINDOW 0x00010000 /* Built-in window-only function */
#define SQLITE_FUNC_INTERNAL 0x00040000 /* For use by NestedParse() only */
#define SQLITE_FUNC_DIRECT 0x00080000 /* Not for use in TRIGGERs or VIEWs */
#define SQLITE_FUNC_SUBTYPE 0x00100000 /* Result likely to have sub-type */
/* SQLITE_SUBTYPE 0x00100000 // Consumer of subtypes */
#define SQLITE_FUNC_UNSAFE 0x00200000 /* Function has side effects */
#define SQLITE_FUNC_INLINE 0x00400000 /* Functions implemented in-line */
#define SQLITE_FUNC_BUILTIN 0x00800000 /* This is a built-in function */
/* SQLITE_RESULT_SUBTYPE 0x01000000 // Generator of subtypes */
#define SQLITE_FUNC_ANYORDER 0x08000000 /* count/min/max aggregate */
/* Identifier numbers for each in-line function */
@ -17910,9 +17954,10 @@ struct FuncDestructor {
#define MFUNCTION(zName, nArg, xPtr, xFunc) \
{nArg, SQLITE_FUNC_BUILTIN|SQLITE_FUNC_CONSTANT|SQLITE_UTF8, \
xPtr, 0, xFunc, 0, 0, 0, #zName, {0} }
#define JFUNCTION(zName, nArg, iArg, xFunc) \
{nArg, SQLITE_FUNC_BUILTIN|SQLITE_DETERMINISTIC|\
SQLITE_FUNC_CONSTANT|SQLITE_UTF8, \
#define JFUNCTION(zName, nArg, bUseCache, bWS, bRS, iArg, xFunc) \
{nArg, SQLITE_FUNC_BUILTIN|SQLITE_DETERMINISTIC|SQLITE_FUNC_CONSTANT|\
SQLITE_UTF8|((bUseCache)*SQLITE_FUNC_RUNONLY)|\
((bRS)*SQLITE_SUBTYPE)|((bWS)*SQLITE_RESULT_SUBTYPE), \
SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} }
#define INLINE_FUNC(zName, nArg, iArg, mFlags) \
{nArg, SQLITE_FUNC_BUILTIN|\
@ -29453,7 +29498,7 @@ SQLITE_PRIVATE void sqlite3MemoryBarrier(void){
SQLITE_MEMORY_BARRIER;
#elif defined(__GNUC__)
__sync_synchronize();
#elif MSVC_VERSION>=1300
#elif MSVC_VERSION>=1400
_ReadWriteBarrier();
#elif defined(MemoryBarrier)
MemoryBarrier();
@ -61447,10 +61492,13 @@ act_like_temp_file:
*/
SQLITE_API sqlite3_file *sqlite3_database_file_object(const char *zName){
Pager *pPager;
const char *p;
while( zName[-1]!=0 || zName[-2]!=0 || zName[-3]!=0 || zName[-4]!=0 ){
zName--;
}
pPager = *(Pager**)(zName - 4 - sizeof(Pager*));
p = zName - 4 - sizeof(Pager*);
assert( EIGHT_BYTE_ALIGNMENT(p) );
pPager = *(Pager**)p;
return pPager->fd;
}
@ -83411,7 +83459,7 @@ static int valueFromFunction(
#endif
assert( pFunc );
if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0
|| (pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL)
|| (pFunc->funcFlags & (SQLITE_FUNC_NEEDCOLL|SQLITE_FUNC_RUNONLY))!=0
){
return SQLITE_OK;
}
@ -89952,6 +90000,18 @@ SQLITE_API void sqlite3_result_subtype(sqlite3_context *pCtx, unsigned int eSubt
#ifdef SQLITE_ENABLE_API_ARMOR
if( pCtx==0 ) return;
#endif
#if defined(SQLITE_STRICT_SUBTYPE) && SQLITE_STRICT_SUBTYPE+0!=0
if( pCtx->pFunc!=0
&& (pCtx->pFunc->funcFlags & SQLITE_RESULT_SUBTYPE)==0
){
char zErr[200];
sqlite3_snprintf(sizeof(zErr), zErr,
"misuse of sqlite3_result_subtype() by %s()",
pCtx->pFunc->zName);
sqlite3_result_error(pCtx, zErr, -1);
return;
}
#endif /* SQLITE_STRICT_SUBTYPE */
pOut = pCtx->pOut;
assert( sqlite3_mutex_held(pOut->db->mutex) );
pOut->eSubtype = eSubtype & 0xff;
@ -100321,7 +100381,7 @@ case OP_VCheck: { /* out2 */
pTab = pOp->p4.pTab;
assert( pTab!=0 );
assert( IsVirtual(pTab) );
assert( pTab->u.vtab.p!=0 );
if( pTab->u.vtab.p==0 ) break;
pVtab = pTab->u.vtab.p->pVtab;
assert( pVtab!=0 );
pModule = pVtab->pModule;
@ -113917,8 +113977,8 @@ SQLITE_PRIVATE int sqlite3ExprListCompare(const ExprList *pA, const ExprList *pB
*/
SQLITE_PRIVATE int sqlite3ExprCompareSkip(Expr *pA,Expr *pB, int iTab){
return sqlite3ExprCompare(0,
sqlite3ExprSkipCollateAndLikely(pA),
sqlite3ExprSkipCollateAndLikely(pB),
sqlite3ExprSkipCollate(pA),
sqlite3ExprSkipCollate(pB),
iTab);
}
@ -147605,10 +147665,11 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){
SrcList *pTabList;
SrcItem *pFrom;
assert( p->selFlags & SF_Resolved );
if( p->selFlags & SF_HasTypeInfo ) return;
p->selFlags |= SF_HasTypeInfo;
pParse = pWalker->pParse;
testcase( (p->selFlags & SF_Resolved)==0 );
assert( (p->selFlags & SF_Resolved) || IN_RENAME_OBJECT );
pTabList = p->pSrc;
for(i=0, pFrom=pTabList->a; i<pTabList->nSrc; i++, pFrom++){
Table *pTab = pFrom->pTab;
@ -148630,6 +148691,7 @@ SQLITE_PRIVATE int sqlite3Select(
TREETRACE(0x1000,pParse,p,
("LEFT-JOIN simplifies to JOIN on term %d\n",i));
pItem->fg.jointype &= ~(JT_LEFT|JT_OUTER);
unsetJoinExpr(p->pWhere, pItem->iCursor, 0);
}
}
if( pItem->fg.jointype & JT_LTORJ ){
@ -148644,17 +148706,15 @@ SQLITE_PRIVATE int sqlite3Select(
TREETRACE(0x1000,pParse,p,
("RIGHT-JOIN simplifies to JOIN on term %d\n",j));
pI2->fg.jointype &= ~(JT_RIGHT|JT_OUTER);
unsetJoinExpr(p->pWhere, pI2->iCursor, 1);
}
}
}
for(j=pTabList->nSrc-1; j>=i; j--){
for(j=pTabList->nSrc-1; j>=0; j--){
pTabList->a[j].fg.jointype &= ~JT_LTORJ;
if( pTabList->a[j].fg.jointype & JT_RIGHT ) break;
}
}
assert( pItem->iCursor>=0 );
unsetJoinExpr(p->pWhere, pItem->iCursor,
pTabList->a[0].fg.jointype & JT_LTORJ);
}
/* No further action if this term of the FROM clause is not a subquery */
@ -166058,6 +166118,20 @@ static SQLITE_NOINLINE void whereAddIndexedExpr(
continue;
}
if( sqlite3ExprIsConstant(pExpr) ) continue;
if( pExpr->op==TK_FUNCTION ){
/* Functions that might set a subtype should not be replaced by the
** value taken from an expression index since the index omits the
** subtype. https://sqlite.org/forum/forumpost/68d284c86b082c3e */
int n;
FuncDef *pDef;
sqlite3 *db = pParse->db;
assert( ExprUseXList(pExpr) );
n = pExpr->x.pList ? pExpr->x.pList->nExpr : 0;
pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0);
if( pDef==0 || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){
continue;
}
}
p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr));
if( p==0 ) break;
p->pIENext = pParse->pIdxEpr;
@ -168240,7 +168314,7 @@ SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){
assert( ExprUseXList(pWin->pOwner) );
assert( pWin->pWFunc!=0 );
pArgs = pWin->pOwner->x.pList;
if( pWin->pWFunc->funcFlags & SQLITE_FUNC_SUBTYPE ){
if( pWin->pWFunc->funcFlags & SQLITE_SUBTYPE ){
selectWindowRewriteEList(pParse, pMWin, pSrc, pArgs, pTab, &pSublist);
pWin->iArgCol = (pSublist ? pSublist->nExpr : 0);
pWin->bExprArgs = 1;
@ -179412,7 +179486,7 @@ SQLITE_PRIVATE int sqlite3CreateFunc(
assert( SQLITE_FUNC_CONSTANT==SQLITE_DETERMINISTIC );
assert( SQLITE_FUNC_DIRECT==SQLITE_DIRECTONLY );
extraFlags = enc & (SQLITE_DETERMINISTIC|SQLITE_DIRECTONLY|
SQLITE_SUBTYPE|SQLITE_INNOCUOUS);
SQLITE_SUBTYPE|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE);
enc &= (SQLITE_FUNC_ENCMASK|SQLITE_ANY);
/* The SQLITE_INNOCUOUS flag is the same bit as SQLITE_FUNC_UNSAFE. But
@ -202993,13 +203067,19 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){
zIn++;
N -= 2;
while( N>0 ){
for(i=0; i<N && zIn[i]!='\\'; i++){}
for(i=0; i<N && zIn[i]!='\\' && zIn[i]!='"'; i++){}
if( i>0 ){
jsonAppendRawNZ(p, zIn, i);
zIn += i;
N -= i;
if( N==0 ) break;
}
if( zIn[0]=='"' ){
jsonAppendRawNZ(p, "\\\"", 2);
zIn++;
N--;
continue;
}
assert( zIn[0]=='\\' );
switch( (u8)zIn[1] ){
case '\'':
@ -203394,7 +203474,8 @@ static void jsonReturnJson(
JsonParse *pParse, /* The complete JSON */
JsonNode *pNode, /* Node to return */
sqlite3_context *pCtx, /* Return value for this function */
int bGenerateAlt /* Also store the rendered text in zAlt */
int bGenerateAlt, /* Also store the rendered text in zAlt */
int omitSubtype /* Do not call sqlite3_result_subtype() */
){
JsonString s;
if( pParse->oom ){
@ -203409,7 +203490,7 @@ static void jsonReturnJson(
pParse->nAlt = s.nUsed;
}
jsonResult(&s);
sqlite3_result_subtype(pCtx, JSON_SUBTYPE);
if( !omitSubtype ) sqlite3_result_subtype(pCtx, JSON_SUBTYPE);
}
}
@ -203450,7 +203531,8 @@ static u32 jsonHexToInt4(const char *z){
static void jsonReturn(
JsonParse *pParse, /* Complete JSON parse tree */
JsonNode *pNode, /* Node to return */
sqlite3_context *pCtx /* Return value for this function */
sqlite3_context *pCtx, /* Return value for this function */
int omitSubtype /* Do not call sqlite3_result_subtype() */
){
switch( pNode->eType ){
default: {
@ -203596,7 +203678,7 @@ static void jsonReturn(
}
case JSON_ARRAY:
case JSON_OBJECT: {
jsonReturnJson(pParse, pNode, pCtx, 0);
jsonReturnJson(pParse, pNode, pCtx, 0, omitSubtype);
break;
}
}
@ -204948,7 +205030,7 @@ static void jsonParseFunc(
printf("iSubst = %u\n", p->iSubst);
printf("iHold = %u\n", p->iHold);
jsonDebugPrintNodeEntries(p->aNode, p->nNode);
jsonReturnJson(p, p->aNode, ctx, 1);
jsonReturnJson(p, p->aNode, ctx, 1, 0);
}
/*
@ -205134,15 +205216,14 @@ static void jsonExtractFunc(
}
if( pNode ){
if( flags & JSON_JSON ){
jsonReturnJson(p, pNode, ctx, 0);
jsonReturnJson(p, pNode, ctx, 0, 0);
}else{
jsonReturn(p, pNode, ctx);
sqlite3_result_subtype(ctx, 0);
jsonReturn(p, pNode, ctx, 1);
}
}
}else{
pNode = jsonLookup(p, zPath, 0, ctx);
if( p->nErr==0 && pNode ) jsonReturn(p, pNode, ctx);
if( p->nErr==0 && pNode ) jsonReturn(p, pNode, ctx, 0);
}
}else{
/* Two or more PATH arguments results in a JSON array with each
@ -205268,7 +205349,7 @@ static void jsonPatchFunc(
if( pResult && pX->oom==0 ){
jsonDebugPrintParse(pX);
jsonDebugPrintNode(pResult);
jsonReturnJson(pX, pResult, ctx, 0);
jsonReturnJson(pX, pResult, ctx, 0, 0);
}else{
sqlite3_result_error_nomem(ctx);
}
@ -205347,7 +205428,7 @@ static void jsonRemoveFunc(
}
}
if( (pParse->aNode[0].jnFlags & JNODE_REMOVE)==0 ){
jsonReturnJson(pParse, pParse->aNode, ctx, 1);
jsonReturnJson(pParse, pParse->aNode, ctx, 1, 0);
}
remove_done:
jsonDebugPrintParse(p);
@ -205476,7 +205557,7 @@ static void jsonReplaceFunc(
jsonReplaceNode(ctx, pParse, (u32)(pNode - pParse->aNode), argv[i+1]);
}
}
jsonReturnJson(pParse, pParse->aNode, ctx, 1);
jsonReturnJson(pParse, pParse->aNode, ctx, 1, 0);
replace_err:
jsonDebugPrintParse(pParse);
jsonParseFree(pParse);
@ -205530,7 +205611,7 @@ static void jsonSetFunc(
}
}
jsonDebugPrintParse(pParse);
jsonReturnJson(pParse, pParse->aNode, ctx, 1);
jsonReturnJson(pParse, pParse->aNode, ctx, 1, 0);
jsonSetDone:
jsonParseFree(pParse);
}
@ -206045,7 +206126,7 @@ static int jsonEachColumn(
case JEACH_KEY: {
if( p->i==0 ) break;
if( p->eType==JSON_OBJECT ){
jsonReturn(&p->sParse, pThis, ctx);
jsonReturn(&p->sParse, pThis, ctx, 0);
}else if( p->eType==JSON_ARRAY ){
u32 iKey;
if( p->bRecursive ){
@ -206061,7 +206142,7 @@ static int jsonEachColumn(
}
case JEACH_VALUE: {
if( pThis->jnFlags & JNODE_LABEL ) pThis++;
jsonReturn(&p->sParse, pThis, ctx);
jsonReturn(&p->sParse, pThis, ctx, 0);
break;
}
case JEACH_TYPE: {
@ -206072,7 +206153,7 @@ static int jsonEachColumn(
case JEACH_ATOM: {
if( pThis->jnFlags & JNODE_LABEL ) pThis++;
if( pThis->eType>=JSON_ARRAY ) break;
jsonReturn(&p->sParse, pThis, ctx);
jsonReturn(&p->sParse, pThis, ctx, 0);
break;
}
case JEACH_ID: {
@ -206365,34 +206446,43 @@ static sqlite3_module jsonTreeModule = {
SQLITE_PRIVATE void sqlite3RegisterJsonFunctions(void){
#ifndef SQLITE_OMIT_JSON
static FuncDef aJsonFunc[] = {
JFUNCTION(json, 1, 0, jsonRemoveFunc),
JFUNCTION(json_array, -1, 0, jsonArrayFunc),
JFUNCTION(json_array_length, 1, 0, jsonArrayLengthFunc),
JFUNCTION(json_array_length, 2, 0, jsonArrayLengthFunc),
JFUNCTION(json_error_position,1, 0, jsonErrorFunc),
JFUNCTION(json_extract, -1, 0, jsonExtractFunc),
JFUNCTION(->, 2, JSON_JSON, jsonExtractFunc),
JFUNCTION(->>, 2, JSON_SQL, jsonExtractFunc),
JFUNCTION(json_insert, -1, 0, jsonSetFunc),
JFUNCTION(json_object, -1, 0, jsonObjectFunc),
JFUNCTION(json_patch, 2, 0, jsonPatchFunc),
JFUNCTION(json_quote, 1, 0, jsonQuoteFunc),
JFUNCTION(json_remove, -1, 0, jsonRemoveFunc),
JFUNCTION(json_replace, -1, 0, jsonReplaceFunc),
JFUNCTION(json_set, -1, JSON_ISSET, jsonSetFunc),
JFUNCTION(json_type, 1, 0, jsonTypeFunc),
JFUNCTION(json_type, 2, 0, jsonTypeFunc),
JFUNCTION(json_valid, 1, 0, jsonValidFunc),
#if SQLITE_DEBUG
JFUNCTION(json_parse, 1, 0, jsonParseFunc),
JFUNCTION(json_test1, 1, 0, jsonTest1Func),
/* calls sqlite3_result_subtype() */
/* | */
/* Uses cache ______ | __ calls sqlite3_value_subtype() */
/* | | | */
/* Num args _________ | | | ___ Flags */
/* | | | | | */
/* | | | | | */
JFUNCTION(json, 1, 1, 1, 0, 0, jsonRemoveFunc),
JFUNCTION(json_array, -1, 0, 1, 1, 0, jsonArrayFunc),
JFUNCTION(json_array_length, 1, 1, 0, 0, 0, jsonArrayLengthFunc),
JFUNCTION(json_array_length, 2, 1, 0, 0, 0, jsonArrayLengthFunc),
JFUNCTION(json_error_position,1, 1, 0, 0, 0, jsonErrorFunc),
JFUNCTION(json_extract, -1, 1, 1, 0, 0, jsonExtractFunc),
JFUNCTION(->, 2, 1, 1, 0, JSON_JSON, jsonExtractFunc),
JFUNCTION(->>, 2, 1, 0, 0, JSON_SQL, jsonExtractFunc),
JFUNCTION(json_insert, -1, 1, 1, 1, 0, jsonSetFunc),
JFUNCTION(json_object, -1, 0, 1, 1, 0, jsonObjectFunc),
JFUNCTION(json_patch, 2, 1, 1, 0, 0, jsonPatchFunc),
JFUNCTION(json_quote, 1, 0, 1, 1, 0, jsonQuoteFunc),
JFUNCTION(json_remove, -1, 1, 1, 0, 0, jsonRemoveFunc),
JFUNCTION(json_replace, -1, 1, 1, 1, 0, jsonReplaceFunc),
JFUNCTION(json_set, -1, 1, 1, 1, JSON_ISSET, jsonSetFunc),
JFUNCTION(json_type, 1, 1, 0, 0, 0, jsonTypeFunc),
JFUNCTION(json_type, 2, 1, 0, 0, 0, jsonTypeFunc),
JFUNCTION(json_valid, 1, 1, 0, 0, 0, jsonValidFunc),
#ifdef SQLITE_DEBUG
JFUNCTION(json_parse, 1, 1, 1, 0, 0, jsonParseFunc),
JFUNCTION(json_test1, 1, 1, 0, 1, 0, jsonTest1Func),
#endif
WAGGREGATE(json_group_array, 1, 0, 0,
jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse,
SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC),
SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8|
SQLITE_DETERMINISTIC),
WAGGREGATE(json_group_object, 2, 0, 0,
jsonObjectStep, jsonObjectFinal, jsonObjectValue, jsonGroupInverse,
SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC)
SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8|
SQLITE_DETERMINISTIC)
};
sqlite3InsertBuiltinFuncs(aJsonFunc, ArraySize(aJsonFunc));
#endif
@ -236129,10 +236219,8 @@ static Fts5HashEntry *fts5HashEntryMerge(
}
/*
** Extract all tokens from hash table iHash and link them into a list
** in sorted order. The hash table is cleared before returning. It is
** the responsibility of the caller to free the elements of the returned
** list.
** Link all tokens from hash table iHash into a list in sorted order. The
** tokens are not removed from the hash table.
*/
static int fts5HashEntrySort(
Fts5Hash *pHash,
@ -238998,6 +239086,14 @@ static void fts5SegIterHashInit(
pLeaf->p = (u8*)pList;
}
}
/* The call to sqlite3Fts5HashScanInit() causes the hash table to
** fill the size field of all existing position lists. This means they
** can no longer be appended to. Since the only scenario in which they
** can be appended to is if the previous operation on this table was
** a DELETE, by clearing the Fts5Index.bDelete flag we can avoid this
** possibility altogether. */
p->bDelete = 0;
}else{
p->rc = sqlite3Fts5HashQuery(p->pHash, sizeof(Fts5Data),
(const char*)pTerm, nTerm, (void**)&pLeaf, &nList
@ -240675,7 +240771,7 @@ static void fts5WriteAppendPoslistData(
const u8 *a = aData;
int n = nData;
assert( p->pConfig->pgsz>0 );
assert( p->pConfig->pgsz>0 || p->rc!=SQLITE_OK );
while( p->rc==SQLITE_OK
&& (pPage->buf.n + pPage->pgidx.n + n)>=p->pConfig->pgsz
){
@ -241935,8 +242031,9 @@ static int sqlite3Fts5IndexOptimize(Fts5Index *p){
assert( p->rc==SQLITE_OK );
fts5IndexFlush(p);
assert( p->nContentlessDelete==0 );
assert( p->rc!=SQLITE_OK || p->nContentlessDelete==0 );
pStruct = fts5StructureRead(p);
assert( p->rc!=SQLITE_OK || pStruct!=0 );
fts5StructureInvalidate(p);
if( pStruct ){
@ -247513,7 +247610,7 @@ static void fts5SourceIdFunc(
){
assert( nArg==0 );
UNUSED_PARAM2(nArg, apUnused);
sqlite3_result_text(pCtx, "fts5: 2023-11-01 11:23:50 17129ba1ff7f0daf37100ee82d507aef7827cf38de1866e2633096ae6ad81301", -1, SQLITE_TRANSIENT);
sqlite3_result_text(pCtx, "fts5: 2023-11-22 14:18:12 d295f48e8f367b066b881780c98bdf980a1d550397d5ba0b0e49842c95b3e8b4", -1, SQLITE_TRANSIENT);
}
/*

67
src/database/sqlite3.h
View File

@ -146,9 +146,9 @@ extern "C" {
** [sqlite3_libversion_number()], [sqlite3_sourceid()],
** [sqlite_version()] and [sqlite_source_id()].
*/
#define SQLITE_VERSION "3.44.0"
#define SQLITE_VERSION_NUMBER 3044000
#define SQLITE_SOURCE_ID "2023-11-01 11:23:50 17129ba1ff7f0daf37100ee82d507aef7827cf38de1866e2633096ae6ad81301"
#define SQLITE_VERSION "3.44.1"
#define SQLITE_VERSION_NUMBER 3044001
#define SQLITE_SOURCE_ID "2023-11-22 14:18:12 d295f48e8f367b066b881780c98bdf980a1d550397d5ba0b0e49842c95b3e8b4"
/*
** CAPI3REF: Run-Time Library Version Numbers
@ -5573,13 +5573,27 @@ SQLITE_API int sqlite3_create_window_function(
** </dd>
**
** [[SQLITE_SUBTYPE]] <dt>SQLITE_SUBTYPE</dt><dd>
** The SQLITE_SUBTYPE flag indicates to SQLite that a function may call
** The SQLITE_SUBTYPE flag indicates to SQLite that a function might call
** [sqlite3_value_subtype()] to inspect the sub-types of its arguments.
** Specifying this flag makes no difference for scalar or aggregate user
** functions. However, if it is not specified for a user-defined window
** function, then any sub-types belonging to arguments passed to the window
** function may be discarded before the window function is called (i.e.
** sqlite3_value_subtype() will always return 0).
** This flag instructs SQLite to omit some corner-case optimizations that
** might disrupt the operation of the [sqlite3_value_subtype()] function,
** causing it to return zero rather than the correct subtype().
** SQL functions that invokes [sqlite3_value_subtype()] should have this
** property. If the SQLITE_SUBTYPE property is omitted, then the return
** value from [sqlite3_value_subtype()] might sometimes be zero even though
** a non-zero subtype was specified by the function argument expression.
**
** [[SQLITE_RESULT_SUBTYPE]] <dt>SQLITE_RESULT_SUBTYPE</dt><dd>
** The SQLITE_RESULT_SUBTYPE flag indicates to SQLite that a function might call
** [sqlite3_result_subtype()] to cause a sub-type to be associated with its
** result.
** Every function that invokes [sqlite3_result_subtype()] should have this
** property. If it does not, then the call to [sqlite3_result_subtype()]
** might become a no-op if the function is used as term in an
** [expression index]. On the other hand, SQL functions that never invoke
** [sqlite3_result_subtype()] should avoid setting this property, as the
** purpose of this property is to disable certain optimizations that are
** incompatible with subtypes.
** </dd>
** </dl>
*/
@ -5587,6 +5601,7 @@ SQLITE_API int sqlite3_create_window_function(
#define SQLITE_DIRECTONLY 0x000080000
#define SQLITE_SUBTYPE 0x000100000
#define SQLITE_INNOCUOUS 0x000200000
#define SQLITE_RESULT_SUBTYPE 0x001000000
/*
** CAPI3REF: Deprecated Functions
@ -5783,6 +5798,12 @@ SQLITE_API int sqlite3_value_encoding(sqlite3_value*);
** information can be used to pass a limited amount of context from
** one SQL function to another. Use the [sqlite3_result_subtype()]
** routine to set the subtype for the return value of an SQL function.
**
** Every [application-defined SQL function] that invoke this interface
** should include the [SQLITE_SUBTYPE] property in the text
** encoding argument when the function is [sqlite3_create_function|registered].
** If the [SQLITE_SUBTYPE] property is omitted, then sqlite3_value_subtype()
** might return zero instead of the upstream subtype in some corner cases.
*/
SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value*);
@ -5913,14 +5934,22 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*);
** <li> ^(when sqlite3_set_auxdata() is invoked again on the same
** parameter)^, or
** <li> ^(during the original sqlite3_set_auxdata() call when a memory
** allocation error occurs.)^ </ul>
** allocation error occurs.)^
** <li> ^(during the original sqlite3_set_auxdata() call if the function
** is evaluated during query planning instead of during query execution,
** as sometimes happens with [SQLITE_ENABLE_STAT4].)^ </ul>
**
** Note the last bullet in particular. The destructor X in
** Note the last two bullets in particular. The destructor X in
** sqlite3_set_auxdata(C,N,P,X) might be called immediately, before the
** sqlite3_set_auxdata() interface even returns. Hence sqlite3_set_auxdata()
** should be called near the end of the function implementation and the
** function implementation should not make any use of P after
** sqlite3_set_auxdata() has been called.
** sqlite3_set_auxdata() has been called. Furthermore, a call to
** sqlite3_get_auxdata() that occurs immediately after a corresponding call
** to sqlite3_set_auxdata() might still return NULL if an out-of-memory
** condition occurred during the sqlite3_set_auxdata() call or if the
** function is being evaluated during query planning rather than during
** query execution.
**
** ^(In practice, auxiliary data is preserved between function calls for
** function parameters that are compile-time constants, including literal
@ -6194,6 +6223,20 @@ SQLITE_API int sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n);
** higher order bits are discarded.
** The number of subtype bytes preserved by SQLite might increase
** in future releases of SQLite.
**
** Every [application-defined SQL function] that invokes this interface
** should include the [SQLITE_RESULT_SUBTYPE] property in its
** text encoding argument when the SQL function is
** [sqlite3_create_function|registered]. If the [SQLITE_RESULT_SUBTYPE]
** property is omitted from the function that invokes sqlite3_result_subtype(),
** then in some cases the sqlite3_result_subtype() might fail to set
** the result subtype.
**
** If SQLite is compiled with -DSQLITE_STRICT_SUBTYPE=1, then any
** SQL function that invokes the sqlite3_result_subtype() interface
** and that does not have the SQLITE_RESULT_SUBTYPE property will raise
** an error. Future versions of SQLite might enable -DSQLITE_STRICT_SUBTYPE=1
** by default.
*/
SQLITE_API void sqlite3_result_subtype(sqlite3_context*,unsigned int);

84
src/files.c
View File

@ -26,14 +26,10 @@
#include <sys/statvfs.h>
// dirname()
#include <libgen.h>
// compression functions
#include "zip/gzip.h"
// sendfile()
#include <fcntl.h>
#include <sys/sendfile.h>
#define BACKUP_DIR "/etc/pihole/config_backups"
// chmod_file() changes the file mode bits of a given file (relative
// to the directory file descriptor) according to mode. mode is an
// octal number representing the bit pattern for the new mode bits
@ -465,14 +461,6 @@ void rotate_files(const char *path, char **first_file)
if(i == 1 && first_file != NULL)
*first_file = strdup(new_path);
size_t old_path_len = strlen(old_path) + 4;
char *old_path_compressed = calloc(old_path_len, sizeof(char));
snprintf(old_path_compressed, old_path_len, "%s.gz", old_path);
size_t new_path_len = strlen(new_path) + 4;
char *new_path_compressed = calloc(new_path_len, sizeof(char));
snprintf(new_path_compressed, new_path_len, "%s.gz", new_path);
if(file_exists(old_path))
{
// Copy file to backup directory
@ -507,46 +495,11 @@ void rotate_files(const char *path, char **first_file)
// Change ownership of file to pihole user
chown_pihole(new_path);
// Compress file if we are rotating a sufficiently old file
if(i > ZIP_ROTATIONS)
{
log_debug(DEBUG_CONFIG, "Compressing %s -> %s",
new_path, new_path_compressed);
if(deflate_file(new_path, new_path_compressed, false))
{
// On success, we remove the uncompressed file
remove(new_path);
}
// Change ownership of file to pihole user
chown_pihole(new_path_compressed);
}
}
else if(file_exists(old_path_compressed))
{
// Rename file
if(rename(old_path_compressed, new_path_compressed) < 0)
{
log_warn("Rotation %s -(MOVE)> %s failed: %s",
old_path_compressed, new_path_compressed, strerror(errno));
}
else
{
// Log success if debug is enabled
log_debug(DEBUG_CONFIG, "Rotated %s -> %s",
old_path_compressed, new_path_compressed);
}
// Change ownership of file to pihole user
chown_pihole(new_path_compressed);
}
// Free memory
free(old_path);
free(new_path);
free(old_path_compressed);
free(new_path_compressed);
}
}
@ -688,3 +641,40 @@ bool files_different(const char *pathA, const char* pathB, unsigned int from)
return different;
}
// Create SHA256 checksum of a file
bool sha256sum(const char *path, uint8_t checksum[SHA256_DIGEST_SIZE])
{
// Open file
FILE *fp = fopen(path, "rb");
if(fp == NULL)
{
log_warn("sha256_file(): Failed to open \"%s\" for reading: %s", path, strerror(errno));
return false;
}
// Initialize SHA2-256 context
struct sha256_ctx ctx;
sha256_init(&ctx);
// Read file in chunks of <pagesize> bytes
const size_t pagesize = getpagesize();
unsigned char *buf = calloc(pagesize, sizeof(char));
size_t len;
while((len = fread(buf, sizeof(char), pagesize, fp)) > 0)
{
// Update SHA256 context
sha256_update(&ctx, len, buf);
}
// Finalize SHA256 context
sha256_digest(&ctx, SHA256_DIGEST_SIZE, checksum);
// Close file
fclose(fp);
// Free memory
free(buf);
return true;
}

5
src/files.h
View File

@ -14,9 +14,11 @@
#include <sys/stat.h>
// setmntent()
#include <mntent.h>
// SHA256_DIGEST_SIZE
#include <nettle/sha2.h>
#define ZIP_ROTATIONS 3
#define MAX_ROTATIONS 15
#define BACKUP_DIR "/etc/pihole/config_backups"
bool chmod_file(const char *filename, const mode_t mode);
bool file_exists(const char *filename);
@ -31,6 +33,7 @@ struct mntent *get_filesystem_details(const char *path);
bool directory_exists(const char *path);
void rotate_files(const char *path, char **first_file);
bool files_different(const char *pathA, const char* pathB, unsigned int from);
bool sha256sum(const char *path, uint8_t checksum[SHA256_DIGEST_SIZE]);
int parse_line(char *line, char **key, char **value);

1
src/gc.c
View File

@ -374,7 +374,6 @@ void *GC_thread(void *val)
if(check_inotify_event())
{
// Reload config
log_info("Reloading config due to pihole.toml change");
reread_config();
}

3
src/main.c
View File

@ -77,7 +77,8 @@ int main (int argc, char *argv[])
// Process pihole.toml configuration file
// The file is rewritten after parsing to ensure that all
// settings are present and have a valid value
readFTLconf(&config, true);
if(readFTLconf(&config, true))
log_info("Parsed config file "GLOBALTOMLPATH" successfully");
// Set process priority
set_nice();

2
src/zip/teleporter.c
View File

@ -312,7 +312,7 @@ static const char *test_and_import_pihole_toml(void *ptr, size_t size, char * co
// a temporary config struct (teleporter_config)
struct config teleporter_config = { 0 };
duplicate_config(&teleporter_config, &config);
if(!readFTLtoml(NULL, &teleporter_config, toml, true, NULL))
if(!readFTLtoml(NULL, &teleporter_config, toml, true, NULL, 0))
return "File etc/pihole/pihole.toml in ZIP archive contains invalid TOML configuration";
// Test dnsmasq config in the imported configuration

2
test/pihole.toml
View File

@ -560,7 +560,7 @@
#
# Possible values are:
# comma-separated list of <[ip_address:]port>
port = "80,[::]:80,443s"
port = "80,[::]:80,443s,[::]:443s"
[webserver.session]
# Session timeout in seconds. If a session is inactive for more than this time, it will

32
test/test_suite.bats
View File

@ -489,21 +489,17 @@
[[ ${lines[0]} == "The Pi-hole FTL engine - "* ]]
}
#@test "No WARNING messages in FTL.log (besides known capability issues)" {
# run bash -c 'grep "WARNING" /var/log/pihole/FTL.log'
# printf "%s\n" "${lines[@]}"
# run bash -c 'grep "WARNING" /var/log/pihole/FTL.log | grep -c -v -E "CAP_NET_ADMIN|CAP_NET_RAW|CAP_SYS_NICE|CAP_IPC_LOCK|CAP_CHOWN"'
# printf "%s\n" "${lines[@]}"
# [[ ${lines[0]} == "0" ]]
#}
@test "No WARNING messages in FTL.log (besides known capability issues)" {
run bash -c 'grep "WARNING:" /var/log/pihole/FTL.log | grep -v -E "CAP_NET_ADMIN|CAP_NET_RAW|CAP_SYS_NICE|CAP_IPC_LOCK|CAP_CHOWN|CAP_NET_BIND_SERVICE|(Cannot set process priority)"'
printf "%s\n" "${lines[@]}"
[[ "${lines[@]}" == "" ]]
}
#@test "No FATAL messages in FTL.log (besides error due to starting FTL more than once)" {
# run bash -c 'grep "FATAL" /var/log/pihole/FTL.log'
# printf "%s\n" "${lines[@]}"
# run bash -c 'grep "FATAL:" /var/log/pihole/FTL.log | grep -c -v "FATAL: create_shm(): Failed to create shared memory object \"FTL-lock\": File exists"'
# printf "%s\n" "${lines[@]}"
# [[ ${lines[0]} == "0" ]]
#}
@test "No CRIT messages in FTL.log (besides error due to starting FTL more than once)" {
run bash -c 'grep "CRIT:" /var/log/pihole/FTL.log | grep -v "CRIT: Initialization of shared memory failed"'
printf "%s\n" "${lines[@]}"
[[ "${lines[@]}" == "" ]]
}
@test "No \"database not available\" messages in FTL.log" {
run bash -c 'grep -c "database not available" /var/log/pihole/FTL.log'
@ -1524,6 +1520,12 @@
[[ $status == 0 ]]
}
@test "SHA256 checksum working" {
run bash -c './pihole-FTL sha256sum test/test.pem'
printf "%s\n" "${lines[@]}"
[[ ${lines[0]} == "eae293f0c30369935a7457a789658bedebf92d544e7526bc43aa07883a597fa9 test/test.pem" ]]
}
@test "API validation" {
run python3 test/api/checkAPI.py
printf "%s\n" "${lines[@]}"
@ -1548,7 +1550,7 @@
[[ "${lines[0]}" == "[ 1.1.1.1 abc-custom.com def-custom.de, 2.2.2.2 äste.com steä.com ]" ]]
run bash -c './pihole-FTL --config webserver.port'
printf "%s\n" "${lines[@]}"
[[ "${lines[0]}" == "80,[::]:80,443s" ]]
[[ "${lines[0]}" == "80,[::]:80,443s,[::]:443s" ]]
}
@test "Create, verify and re-import Teleporter file via CLI" {