Merge e7eaff5377 into e10bb5c605

More recent versions of GCC try to check for use after free. Realloc
will take the input pointer, `ptr_in` and with free it after a successful
allocation. However, it will not change the contents of the pointer, and
thus GCC in some cases thinks it can/could be used. By explicitly
setting `ptr_in` to NULL after an successful allocation, we can keep GCC
happy and our codebase sane.

Signed-off-by: Olliver Schinagl <oliver@schinagl.nl>

.devcontainer/devcontainer.json

@@ -1,10 +1,20 @@
 {
-  "name": "FTL x86_64 Build Env",  
-  "image": "ghcr.io/pi-hole/ftl-build:x86_64", 
-  "extensions": [
-    "jetmartin.bats",
-    "ms-vscode.cpptools",
-    "ms-vscode.cmake-tools",
-    "eamodio.gitlens"
-  ],  
-}
+  "name": "FTL x86_64 Build Env",
+  "image": "ghcr.io/pi-hole/ftl-build:v2.5",
+  "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ],
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "jetmartin.bats",
+        "ms-vscode.cpptools",
+        "ms-vscode.cmake-tools",
+        "eamodio.gitlens"
+      ]
+    }
+  },
+  "mounts": [
+    "type=bind,source=/home/${localEnv:USER}/.ssh,target=/root/.ssh,readonly",
+    "type=bind,source=/var/www/html,target=/var/www/html,readonly"
+  ]
+
+}

.github/.codespellignore

@@ -2,5 +2,8 @@ ssudo
 tre
 ede
 nd
+doubleclick
 requestor
-requestors
+requestors
+punycode
+bitap

.github/.codespellignore_lines

@@ -0,0 +1 @@
+			self.errors.append("Exception when GETing from FTL: " + str(e))

.github/Dockerfile

@@ -0,0 +1,32 @@
+FROM ghcr.io/pi-hole/ftl-build:v2.5 AS builder
+
+WORKDIR /app
+
+COPY . /app
+
+ARG CI_ARCH="linux/amd64"
+ENV CI_ARCH ${CI_ARCH}
+ARG GIT_BRANCH="test"
+ENV GIT_BRANCH ${GIT_BRANCH}
+ARG GIT_TAG="test"
+ENV GIT_TAG ${GIT_TAG}
+
+# Build FTL
+# Remove possible old build files
+RUN rm -rf cmake && \
+# Build FTL
+    bash build.sh "-DSTATIC=${STATIC}" && \
+# Run binary architecture tests
+    bash test/arch_test.sh && \
+# Run full test suite
+    bash test/run.sh && \
+# Move FTL binary to root directory
+    cd / &&\
+    mv /app/pihole-FTL . && \
+# Create tarball of API docs
+    tar -C /app/src/api/docs/content/ -czvf /api-docs.tar.gz .
+
+# Create final image containing only the FTL binary and API docs
+FROM scratch AS result
+COPY --from=builder /pihole-FTL /pihole-FTL
+COPY --from=builder /api-docs.tar.gz /api-docs.tar.gz

.github/actions/build-and-test/action.yml

@@ -0,0 +1,126 @@
+name: Build and test
+description: Builds and tests FTL on all supported platforms
+
+inputs:
+  platform:
+    required: true
+    description: The platform to build for
+  git_branch:
+    required: true
+    description: The branch to build from
+  git_tag:
+    required: true
+    description: The tag to build from (if any)
+  bin_name:
+    required: true
+    description: The name of the binary to build
+  artifact_name:
+    required: true
+    description: The name of the artifact to upload
+  event_name:
+    required: true
+    description: The name of the event that triggered the workflow run
+  actor:
+    required: true
+    description: The name of the user or app that initiated the workflow run
+  target_dir:
+    required: true
+    description: The directory to deploy the artifacts to
+  # Secrets cannot be accessed in the action.yml file so we need to pass them as
+  # inputs to the action.
+  SSH_KEY:
+    required: true
+    description: The SSH private key to use for authentication
+  KNOWN_HOSTS:
+    required: true
+    description: The SSH known hosts file
+  SSH_USER:
+    required: true
+    description: The SSH user to use for authentication
+  SSH_HOST:
+    required: true
+    description: The SSH host to connect to
+
+# Both the definition of environment variables and checking out the code
+# needs to be done outside of the composite action as
+# - environment variables cannot be defined using inputs
+# - the checkout action needs to be the first step in the workflow, otherwise we
+#   cannot use the composite action as the corresponding "action.yml" isn't
+#   there yet
+runs:
+  using: "composite"
+  steps:
+    -
+      name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3.1.0
+    -
+      name: Print directory contents
+      shell: bash
+      run: ls -l
+    -
+      name: Build and test FTL in ftl-build container (QEMU)
+      uses: Wandalen/wretry.action@v1.4.8
+      with:
+        attempt_limit: 3
+        action: docker/build-push-action@v5.0.0
+        with: |
+          platforms: ${{ inputs.platform }}
+          pull: true
+          push: false
+          context: .
+          target: result
+          file: .github/Dockerfile
+          outputs: |
+            type=tar,dest=build.tar
+          build-args: |
+            "CI_ARCH=${{ inputs.platform }}"
+            "GIT_BRANCH=${{ inputs.git_branch }}"
+            "GIT_TAG=${{ inputs.git_tag }}"
+    -
+      name: List files in current directory
+      shell: bash
+      run: ls -l
+    -
+      name: Extract FTL binary from container
+      shell: bash
+      run: |
+        tar -xf build.tar pihole-FTL
+    -
+      name: "Generate checksum file"
+      shell: bash
+      run: |
+        mv pihole-FTL "${{ inputs.bin_name }}"
+        sha1sum pihole-FTL-* > ${{ inputs.bin_name }}.sha1
+    -
+      name: Store binary artifacts for later deployoment
+      if: inputs.event_name != 'pull_request'
+      uses: actions/upload-artifact@v4.3.1
+      with:
+        name: ${{ inputs.artifact_name }}
+        path: '${{ inputs.bin_name }}*'
+    -
+      name: Extract documentation files from container
+      if: inputs.event_name != 'pull_request' && inputs.platform == 'linux/amd64'
+      shell: bash
+      run: |
+        tar -xf build.tar api-docs.tar.gz
+    -
+      name: Upload documentation artifacts for deployoment
+      if: inputs.event_name != 'pull_request' && inputs.platform == 'linux/amd64'
+      uses: actions/upload-artifact@v4.3.1
+      with:
+        name: pihole-api-docs
+        path: 'api-docs.tar.gz'
+    -
+      name: Deploy
+      if: inputs.event_name != 'pull_request'
+      uses: ./.github/actions/deploy
+      with:
+        pattern: ${{ inputs.bin_name }}-binary
+        target_dir: ${{ inputs.target_dir }}
+        event_name: ${{ inputs.event_name }}
+        actor: ${{ inputs.actor }}
+        SSH_KEY: ${{ inputs.SSH_KEY }}
+        KNOWN_HOSTS: ${{ inputs.KNOWN_HOSTS }}
+        SSH_USER: ${{ inputs.SSH_USER }}
+        SSH_HOST: ${{ inputs.SSH_HOST }}

.github/actions/deploy/action.yml

@@ -0,0 +1,96 @@
+name: Deploy
+description: Deploy the FTL binary and documentation
+
+inputs:
+  pattern:
+    required: true
+    description: The pattern to match the artifacts to download
+  target_dir:
+    required: true
+    description: The directory to deploy the artifacts to
+  event_name:
+    required: true
+    description: The name of the event that triggered the workflow run
+  actor:
+    required: true
+    description: The name of the user or app that initiated the workflow run
+  # Secrets cannot be accessed in the action.yml file so we need to pass them as
+  # inputs to the action.
+  SSH_KEY:
+    required: true
+    description: The SSH private key to use for authentication
+  KNOWN_HOSTS:
+    required: true
+    description: The SSH known hosts file
+  SSH_USER:
+    required: true
+    description: The SSH user to use for authentication
+  SSH_HOST:
+    required: true
+    description: The SSH host to connect to
+
+runs:
+  using: "composite"
+  steps:
+    -
+      name: Get binaries built in previous jobs
+      uses: actions/download-artifact@v4.1.4
+      id: download
+      with:
+        path: ftl_builds/
+        pattern: ${{ inputs.pattern }}
+        merge-multiple: true
+    -
+      name: Get documentation files built in previous jobs
+      if: inputs.pattern == 'pihole-FTL-amd64-binary'
+      uses: actions/download-artifact@v4.1.4
+      with:
+        path: ftl_builds/
+        name: pihole-api-docs
+    -
+      name: Display structure of downloaded files
+      shell: bash
+      run: ls -R
+      working-directory: ${{steps.download.outputs.download-path}}
+    -
+      name: Install SSH Key
+      uses: benoitchantre/setup-ssh-authentication-action@1.0.1
+      with:
+        private-key: ${{ inputs.SSH_KEY }}
+        private-key-name: id_rsa
+        known-hosts: ${{ inputs.KNOWN_HOSTS }}
+    -
+      name: Set private key permissions
+      shell: bash
+      run: chmod 600 ~/.ssh/id_rsa
+    -
+      name: Untar documentation files
+      if: inputs.pattern == 'pihole-FTL-amd64-binary'
+      working-directory: ftl_builds/
+      shell: bash
+      run: |
+        mkdir docs/
+        tar xzvf api-docs.tar.gz -C docs/
+    -
+      name: Display structure of files ready for upload
+      working-directory: ftl_builds/
+      shell: bash
+      run: ls -R
+    -
+      name: Transfer Builds to Pi-hole server for pihole checkout
+      if: inputs.actor != 'dependabot[bot]'
+      env:
+        USER: ${{ inputs.SSH_USER }}
+        HOST: ${{ inputs.SSH_HOST }}
+        TARGET_DIR: ${{ inputs.target_dir }}
+        SOURCE_DIR: ftl_builds/
+      shell: bash
+      run: |
+        bash ./deploy.sh
+    -
+      name: Attach binaries to release
+      if: inputs.event_name == 'release'
+      uses: softprops/action-gh-release@v2
+      with:
+        files: |
+          ftl_builds/*

.github/workflows/build.yml

@@ -7,6 +7,7 @@ on:
   pull_request:
   release:
     types: [published]
+  workflow_dispatch:
 
 jobs:
   smoke-tests:
@@ -14,6 +15,7 @@ jobs:
       github.event_name == 'push'
       || github.event_name == 'release'
       || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository)
+      || github.event_name == 'workflow_dispatch'
 
     outputs:
       GIT_TAG: ${{ steps.variables.outputs.GIT_TAG }}
@@ -24,7 +26,7 @@ jobs:
     steps:
       -
         name: Checkout code
-        uses: actions/checkout@v3.6.0
+        uses: actions/checkout@v4.1.4
       -
         name: "Calculate required variables"
         id: variables
@@ -48,123 +50,78 @@ jobs:
           [[ $FAIL == 1 ]] && exit 1 || echo "Branch name depth check passed."
         shell: bash
 
-  build:
+  gha:
     runs-on: ubuntu-latest
-
     needs: smoke-tests
-
-    container: ghcr.io/pi-hole/ftl-build:v1.26-${{ matrix.arch }}
-
     strategy:
       fail-fast: false
-
       matrix:
         include:
-          - arch: x86_64
-            bin_name: pihole-FTL-linux-x86_64
-          - arch: x86_64
-            arch_extra: _full
-            bin_name: pihole-FTL-linux-x86_64_full
-          - arch: x86_64-musl
-            bin_name: pihole-FTL-musl-linux-x86_64
-          - arch: x86_32
-            bin_name: pihole-FTL-linux-x86_32
-          - arch: armv4t
-            bin_name: pihole-FTL-armv4-linux-gnueabi
-          - arch: armv5te
-            bin_name: pihole-FTL-armv5-linux-gnueabi
-          - arch: armv6hf
-            bin_name:  pihole-FTL-armv6-linux-gnueabihf
-          - arch: armv7hf
-            bin_name: pihole-FTL-armv7-linux-gnueabihf
-          - arch: armv8a
-            bin_name: pihole-FTL-armv8-linux-gnueabihf
-          - arch: aarch64
-            bin_name: pihole-FTL-aarch64-linux-gnu
-          - arch: riscv64
-            bin_name: pihole-FTL-riscv64-linux-gnu
-
+          - platform: linux/amd64
+            bin_name: pihole-FTL-amd64
+          - platform: linux/386
+            bin_name: pihole-FTL-386
+          - platform: linux/riscv64
+            bin_name: pihole-FTL-riscv64
     env:
-      CI_ARCH: ${{ matrix.arch }}${{ matrix.arch_extra }}
-
+      CI_ARCH: ${{ matrix.platform }}
+      GIT_BRANCH: ${{ needs.smoke-tests.outputs.GIT_BRANCH }}
+      GIT_TAG: ${{ needs.smoke-tests.outputs.GIT_TAG }}
     steps:
       -
         name: Checkout code
-        uses: actions/checkout@v3.6.0
+        uses: actions/checkout@v4.1.4
       -
-        name: "Fix ownership of repository"
-        run: chown -R root .
-      -
-        name: "Fix ownership of repository"
-        run: chown -R root .
-      -
-        name: "Build"
-        env:
-          GIT_BRANCH: ${{ needs.smoke-tests.outputs.GIT_BRANCH }}
-          GIT_TAG: ${{ needs.smoke-tests.outputs.GIT_TAG }}
-        run: |
-          bash build.sh "-DSTATIC=${STATIC}"
-      -
-        name: "Binary checks"
-        run: |
-          bash test/arch_test.sh
-      -
-        name: "Test x86_32/64 binaries"
-        if: matrix.arch == 'x86_64' || matrix.arch == 'x86_64-musl' || matrix.arch == 'x86_32'
-        run: |
-          bash test/run.sh
-      -
-        name: "Generate checksum file"
-        run: |
-          mv pihole-FTL "${{ matrix.bin_name }}"
-          sha1sum pihole-FTL-* > ${{ matrix.bin_name }}.sha1
-      -
-        name: Upload artifacts to job for later processing
-        if: github.event_name != 'pull_request'
-        uses: actions/upload-artifact@v3.1.3
+        name: Build and test and deploy FTL
+        uses: ./.github/actions/build-and-test
         with:
-          name: tmp-binary-storage
-          path: '${{ matrix.bin_name }}*'
+          platform: ${{ matrix.platform }}
+          bin_name: ${{ matrix.bin_name }}
+          artifact_name: ${{ matrix.bin_name }}-binary
+          target_dir: ${{ needs.smoke-tests.outputs.OUTPUT_DIR }}
+          git_branch: ${{ needs.smoke-tests.outputs.GIT_BRANCH }}
+          git_tag: ${{ needs.smoke-tests.outputs.GIT_TAG }}
+          event_name: ${{ github.event_name }}
+          actor: ${{ github.actor }}
+          SSH_KEY: ${{ secrets.SSH_KEY }}
+          KNOWN_HOSTS: ${{ secrets.KNOWN_HOSTS }}
+          SSH_USER: ${{ secrets.SSH_USER }}
+          SSH_HOST: ${{ secrets.SSH_HOST }}
 
-  deploy:
-    if: github.event_name != 'pull_request'
-    needs: [smoke-tests, build]
-    runs-on: ubuntu-latest
+  self-hosted:
+    runs-on: self-hosted
+    needs: smoke-tests
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/arm/v6
+            bin_name: pihole-FTL-armv6
+          - platform: linux/arm/v7
+            bin_name: pihole-FTL-armv7
+          - platform: linux/arm64/v8
+            bin_name: pihole-FTL-arm64
+    env:
+      CI_ARCH: ${{ matrix.platform }}
+      GIT_BRANCH: ${{ needs.smoke-tests.outputs.GIT_BRANCH }}
+      GIT_TAG: ${{ needs.smoke-tests.outputs.GIT_TAG }}
     steps:
       -
         name: Checkout code
-        uses: actions/checkout@v3.6.0
+        uses: actions/checkout@v4.1.4
       -
-        name: Get Binaries built in previous jobs
-        uses: actions/download-artifact@v3.0.2
-        id: download
+        name: Build and test and deploy FTL
+        uses: ./.github/actions/build-and-test
         with:
-          name: tmp-binary-storage
-          path: ftl-builds/
-      -
-        name: Display structure of downloaded files
-        run: ls -R
-        working-directory: ${{steps.download.outputs.download-path}}
-      -
-        name: Install SSH Key
-        uses: benoitchantre/setup-ssh-authentication-action@1.0.1
-        with:
-          private-key: ${{ secrets.SSH_KEY }}
-          known-hosts: ${{ secrets.KNOWN_HOSTS }}
-      -
-        name: Transfer Builds to Pi-hole server for pihole checkout
-        if: github.actor != 'dependabot[bot]'
-        env:
-          USER: ${{ secrets.SSH_USER }}
-          HOST: ${{ secrets.SSH_HOST }}
-          TARGET_DIR: ${{ needs.smoke-tests.outputs.OUTPUT_DIR }}
-          SOURCE_DIR: ${{ steps.download.outputs.download-path }}
-        run: |
-          bash ./deploy.sh
-      -
-        name: Attach binaries to release
-        if: github.event_name == 'release'
-        uses: softprops/action-gh-release@v1
-        with:
-          files: |
-            ${{ steps.download.outputs.download-path }}/*
+          platform: ${{ matrix.platform }}
+          bin_name: ${{ matrix.bin_name }}
+          artifact_name: ${{ matrix.bin_name }}-binary
+          target_dir: ${{ needs.smoke-tests.outputs.OUTPUT_DIR }}
+          git_branch: ${{ needs.smoke-tests.outputs.GIT_BRANCH }}
+          git_tag: ${{ needs.smoke-tests.outputs.GIT_TAG }}
+          event_name: ${{ github.event_name }}
+          actor: ${{ github.actor }}
+          SSH_KEY: ${{ secrets.SSH_KEY }}
+          KNOWN_HOSTS: ${{ secrets.KNOWN_HOSTS }}
+          SSH_USER: ${{ secrets.SSH_USER }}
+          SSH_HOST: ${{ secrets.SSH_HOST }}

.github/workflows/codespell.yml

@@ -1,5 +1,8 @@
 name: Codespell
 on:
+  push:
+    branches:
+      - '**'
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
 
@@ -10,10 +13,11 @@ jobs:
     steps:
     -
       name: Checkout repository
-      uses: actions/checkout@v3.6.0
+      uses: actions/checkout@v4.1.4
     -
       name: Spell-Checking
       uses: codespell-project/actions-codespell@master
       with:
         ignore_words_file: .github/.codespellignore
-        skip: ./src/database/sqlite3.c,./src/database/sqlite3.h,./src/database/shell.c,./src/lua,./src/dnsmasq,./src/tre-regex,./.git,./test/libs
+        skip: ./src/database/sqlite3.c,./src/database/sqlite3.h,./src/database/shell.c,./src/lua,./src/dnsmasq,./src/tre-regex,./.git,./test/libs,./src/webserver/civetweb,./src/zip/miniz,./src/api/docs/content/external
+        exclude_file: .github/.codespellignore_lines

.github/workflows/merge-conflict.yml

@@ -9,11 +9,11 @@ on:
     types: [synchronize]
 
 jobs:
-  main:
+  merge-conflict:
     runs-on: ubuntu-latest
     steps:
       - name: Check if PRs are have merge conflicts
-        uses: eps1lon/actions-label-merge-conflict@v2.1.0
+        uses: eps1lon/actions-label-merge-conflict@v3.0.0
         with:
           dirtyLabel: "Merge conflicts"
           repoToken: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/openapi-validator.yml

@@ -0,0 +1,26 @@
+name: API validation
+
+on: [push]
+
+env:
+  CI: true
+
+jobs:
+  openapi-validator:
+    name: Node
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v4.1.4
+
+      - name: Set Node.js version
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Install npm dependencies
+        run: npm ci
+
+      - name: Run tests
+        run: npm test

.github/workflows/stale.yml

@@ -40,7 +40,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3.6.0
+        uses: actions/checkout@v4.1.4
       - name: Remove 'stale' label
         run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }}
         env:

.github/workflows/sync-back-to-dev.yml

@@ -11,7 +11,7 @@ jobs:
     name: Syncing branches
     steps:
       - name: Checkout
-        uses: actions/checkout@v3.6.0
+        uses: actions/checkout@v4.1.4
       - name: Opening pull request
         run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal'
         env:

.gitignore

@@ -1,25 +1,40 @@
 # Generated binary
-pihole-FTL
+pihole-FTL*
 
 # Versioning files (generated by Makefile)
-version*
+version.h
+version~
 
 # CMake files generated during compilation
 /cmake/
+/cmake_ci/
 /cmake-build-debug/
 /cmake-build-release/
 
 # IDE files
 .idea/
 *.sw*
+/.vscode
 .vscode/
 /.vscode/
 /build/
 
+# __pycache__ files (API tests)
+__pycache__/
+
+# When patch fails to apply a patch segment to the original file, it saves the
+# temporary original file copy out durably as *.orig, dumps the rejected segment
+# to *.rej, and continues trying to apply patch segments.
+*.orig
+*.rej
+
 # MAC->Vendor database files
 tools/manuf.data
 tools/macvendor.db
 
+# Documentation files generated by cmake
+src/api/docs/hex
+
 # Test dependencies
 /node_modules/
 

CMakeLists.txt

@@ -9,8 +9,9 @@
 # Please see LICENSE file for your rights under this license.
 
 cmake_minimum_required(VERSION 2.8.12)
+
 project(PIHOLE_FTL C)
 
-set(DNSMASQ_VERSION pi-hole-v2.89-9461807)
+set(DNSMASQ_VERSION pi-hole-v2.90+1)
 
 add_subdirectory(src)

build.sh

@@ -12,6 +12,10 @@
 # Abort script if one command returns a non-zero value
 set -e
 
+# Set builddir
+builddir="cmake/"
+
+# Parse arguments
 for var in "$@"
 do
     case "${var}" in
@@ -19,22 +23,39 @@ do
         "-C" | "CLEAN"   ) clean=1 && nobuild=1;;
         "-i" | "install" ) install=1;;
         "-t" | "test"    ) test=1;;
+        "ci"             ) builddir="cmake_ci/";;
     esac
 done
 
 # Prepare build environment
 if [[ -n "${clean}" ]]; then
     echo "Cleaning build environment"
-    rm -rf cmake/
+    # Remove build directory
+    rm -rf "${builddir}"
     if [[ -n ${nobuild} ]]; then
         exit 0
     fi
 fi
 
+# Remove possibly outdated api/docs elements
+for filename in src/api/docs/hex/* src/api/docs/hex/**/*; do
+    # Skip if not a file
+    if [ ! -f "${filename}" ]; then
+        continue
+    fi
+
+    # Get the original filename
+    original_filename="${filename/"src/api/docs/hex/"/"src/api/docs/content/"}"
+
+    # Remove the file if it is outdated
+    if [ "${filename}" -ot "${original_filename}" ]; then
+        rm "${filename}"
+    fi
+done
+
 # Remove compiled LUA scripts if older than the plain ones
 for scriptname in src/lua/scripts/*.lua; do
     if [ -f "${scriptname}.hex" ] && [ "${scriptname}.hex" -ot "${scriptname}" ]; then
-        echo "INFO: ${scriptname} is outdated and will be recompiled"
         rm "${scriptname}.hex"
     fi
 done
@@ -42,18 +63,18 @@ done
 # Configure build, pass CMake CACHE entries if present
 # Wrap multiple options in "" as first argument to ./build.sh:
 #     ./build.sh "-DA=1 -DB=2" install
-mkdir -p cmake
-cd cmake
+mkdir -p "${builddir}"
+cd "${builddir}"
 if [[ "${1}" == "-D"* ]]; then
     cmake "${1}" ..
 else
     cmake ..
 fi
 
-# Build the sources
+# Build the sources with the number of available cores
 cmake --build . -- -j $(nproc)
 
-# If we are asked to install, we do this here
+# If we are asked to install, we do this here (requires root privileges)
 # Otherwise, we simply copy the binary one level up
 if [[ -n "${install}" ]]; then
     echo "Installing pihole-FTL"
@@ -64,6 +85,7 @@ else
     cp pihole-FTL ../
 fi
 
+# If we are asked to run tests, we do this here
 if [[ -n "${test}" ]]; then
     cd ..
     ./test/run.sh

deploy.sh

@@ -62,4 +62,8 @@ for dir in "${path[@]}"; do
 done
 
 sftp -r -b - "${USER}"@"${HOST}" <<< "cd ${old_path}
+-mkdir ./docs
+-mkdir ./docs/external
+-mkdir ./docs/images
+-mkdir ./docs/specs
 put ${SOURCE_DIR}/* ./"

package-lock.json

@@ -0,0 +1,851 @@
+{
+  "name": "pihole-ftl",
+  "version": "1.0.0",
+  "lockfileVersion": 2,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "pihole-ftl",
+      "version": "1.0.0",
+      "license": "EUPL-1.2",
+      "devDependencies": {
+        "openapi-enforcer": "^1.13.1",
+        "openapi-examples-validator": "^4.2.1"
+      }
+    },
+    "node_modules/@apidevtools/json-schema-ref-parser": {
+      "version": "9.0.9",
+      "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.0.9.tgz",
+      "integrity": "sha512-GBD2Le9w2+lVFoc4vswGI/TjkNIZSVp7+9xPf+X3uidBfWnAeUWmquteSyt0+VCrhNMWj/FTABISQrD3Z/YA+w==",
+      "dev": true,
+      "dependencies": {
+        "@jsdevtools/ono": "^7.1.3",
+        "@types/json-schema": "^7.0.6",
+        "call-me-maybe": "^1.0.1",
+        "js-yaml": "^4.1.0"
+      }
+    },
+    "node_modules/@apidevtools/json-schema-ref-parser/node_modules/argparse": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
+      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
+      "dev": true
+    },
+    "node_modules/@apidevtools/json-schema-ref-parser/node_modules/js-yaml": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
+      "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
+      "dev": true,
+      "dependencies": {
+        "argparse": "^2.0.1"
+      },
+      "bin": {
+        "js-yaml": "bin/js-yaml.js"
+      }
+    },
+    "node_modules/@jsdevtools/ono": {
+      "version": "7.1.3",
+      "resolved": "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz",
+      "integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==",
+      "dev": true
+    },
+    "node_modules/@types/json-schema": {
+      "version": "7.0.11",
+      "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz",
+      "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==",
+      "dev": true
+    },
+    "node_modules/ajv": {
+      "version": "6.12.6",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
+      "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+      "dev": true,
+      "dependencies": {
+        "fast-deep-equal": "^3.1.1",
+        "fast-json-stable-stringify": "^2.0.0",
+        "json-schema-traverse": "^0.4.1",
+        "uri-js": "^4.2.2"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/epoberezkin"
+      }
+    },
+    "node_modules/argparse": {
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
+      "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
+      "dev": true,
+      "dependencies": {
+        "sprintf-js": "~1.0.2"
+      }
+    },
+    "node_modules/axios": {
+      "version": "0.21.4",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
+      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
+      "dev": true,
+      "dependencies": {
+        "follow-redirects": "^1.14.0"
+      }
+    },
+    "node_modules/balanced-match": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
+      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
+      "dev": true
+    },
+    "node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
+      "dependencies": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "node_modules/call-me-maybe": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/call-me-maybe/-/call-me-maybe-1.0.1.tgz",
+      "integrity": "sha1-JtII6onje1y95gJQoV8DHBak1ms=",
+      "dev": true
+    },
+    "node_modules/concat-map": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
+      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
+      "dev": true
+    },
+    "node_modules/decimal.js": {
+      "version": "10.2.1",
+      "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.2.1.tgz",
+      "integrity": "sha512-KaL7+6Fw6i5A2XSnsbhm/6B+NuEA7TZ4vqxnd5tXz9sbKtrN9Srj8ab4vKVdK8YAqZO9P1kg45Y6YLoduPf+kw==",
+      "dev": true
+    },
+    "node_modules/errno": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/errno/-/errno-1.0.0.tgz",
+      "integrity": "sha512-3zV5mFS1E8/1bPxt/B0xxzI1snsg3uSCIh6Zo1qKg6iMw93hzPANk9oBFzSFBFrwuVoQuE3rLoouAUfwOAj1wQ==",
+      "dev": true,
+      "dependencies": {
+        "prr": "~1.0.1"
+      },
+      "bin": {
+        "errno": "cli.js"
+      }
+    },
+    "node_modules/esprima": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
+      "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
+      "dev": true,
+      "bin": {
+        "esparse": "bin/esparse.js",
+        "esvalidate": "bin/esvalidate.js"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/fast-deep-equal": {
+      "version": "3.1.3",
+      "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
+      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
+      "dev": true
+    },
+    "node_modules/fast-json-stable-stringify": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
+      "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==",
+      "dev": true
+    },
+    "node_modules/follow-redirects": {
+      "version": "1.15.2",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
+      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://github.com/sponsors/RubenVerborgh"
+        }
+      ],
+      "engines": {
+        "node": ">=4.0"
+      },
+      "peerDependenciesMeta": {
+        "debug": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/foreach": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.6.tgz",
+      "integrity": "sha512-k6GAGDyqLe9JaebCsFCoudPPWfihKu8pylYXRlqP1J7ms39iPoTtk2fviNglIeQEwdh0bQeKJ01ZPyuyQvKzwg==",
+      "dev": true
+    },
+    "node_modules/format-util": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/format-util/-/format-util-1.0.5.tgz",
+      "integrity": "sha512-varLbTj0e0yVyRpqQhuWV+8hlePAgaoFRhNFj50BNjEIrw1/DphHSObtqwskVCPWNgzwPoQrZAbfa/SBiicNeg==",
+      "dev": true
+    },
+    "node_modules/fs.realpath": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
+      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
+      "dev": true
+    },
+    "node_modules/glob": {
+      "version": "7.2.3",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+      "dev": true,
+      "dependencies": {
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.1.1",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      },
+      "engines": {
+        "node": "*"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/inflight": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
+      "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
+      "dev": true,
+      "dependencies": {
+        "once": "^1.3.0",
+        "wrappy": "1"
+      }
+    },
+    "node_modules/inherits": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
+      "dev": true
+    },
+    "node_modules/js-yaml": {
+      "version": "3.14.1",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz",
+      "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==",
+      "dev": true,
+      "dependencies": {
+        "argparse": "^1.0.7",
+        "esprima": "^4.0.0"
+      },
+      "bin": {
+        "js-yaml": "bin/js-yaml.js"
+      }
+    },
+    "node_modules/json-pointer": {
+      "version": "0.6.2",
+      "resolved": "https://registry.npmjs.org/json-pointer/-/json-pointer-0.6.2.tgz",
+      "integrity": "sha512-vLWcKbOaXlO+jvRy4qNd+TI1QUPZzfJj1tpJ3vAXDych5XJf93ftpUKe5pKCrzyIIwgBJcOcCVRUfqQP25afBw==",
+      "dev": true,
+      "dependencies": {
+        "foreach": "^2.0.4"
+      }
+    },
+    "node_modules/json-schema-traverse": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
+      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
+      "dev": true
+    },
+    "node_modules/lodash.clonedeep": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
+      "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=",
+      "dev": true
+    },
+    "node_modules/lodash.flatmap": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/lodash.flatmap/-/lodash.flatmap-4.5.0.tgz",
+      "integrity": "sha1-74y/QI9uSCaGYzRTBcaswLd4cC4=",
+      "dev": true
+    },
+    "node_modules/lodash.flatten": {
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz",
+      "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=",
+      "dev": true
+    },
+    "node_modules/lodash.merge": {
+      "version": "4.6.2",
+      "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
+      "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==",
+      "dev": true
+    },
+    "node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "dev": true,
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/once": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+      "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+      "dev": true,
+      "dependencies": {
+        "wrappy": "1"
+      }
+    },
+    "node_modules/ono": {
+      "version": "4.0.11",
+      "resolved": "https://registry.npmjs.org/ono/-/ono-4.0.11.tgz",
+      "integrity": "sha512-jQ31cORBFE6td25deYeD80wxKBMj+zBmHTrVxnc6CKhx8gho6ipmWM5zj/oeoqioZ99yqBls9Z/9Nss7J26G2g==",
+      "dev": true,
+      "dependencies": {
+        "format-util": "^1.0.3"
+      }
+    },
+    "node_modules/openapi-enforcer": {
+      "version": "1.13.1",
+      "resolved": "https://registry.npmjs.org/openapi-enforcer/-/openapi-enforcer-1.13.1.tgz",
+      "integrity": "sha512-NDDmyonl3bgxP+RabJsTPxn8EEza4Aet5zEocfX6nP9LL0J52aNFMYxNHoAuqoY3zQcZG2pnm3DMi2gYiRYbQg==",
+      "dev": true,
+      "dependencies": {
+        "axios": "^0.21.1",
+        "json-schema-ref-parser": "^6.1.0"
+      }
+    },
+    "node_modules/openapi-enforcer/node_modules/json-schema-ref-parser": {
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/json-schema-ref-parser/-/json-schema-ref-parser-6.1.0.tgz",
+      "integrity": "sha512-pXe9H1m6IgIpXmE5JSb8epilNTGsmTb2iPohAXpOdhqGFbQjNeHHsZxU+C8w6T81GZxSPFLeUoqDJmzxx5IGuw==",
+      "deprecated": "Please switch to @apidevtools/json-schema-ref-parser",
+      "dev": true,
+      "dependencies": {
+        "call-me-maybe": "^1.0.1",
+        "js-yaml": "^3.12.1",
+        "ono": "^4.0.11"
+      }
+    },
+    "node_modules/openapi-examples-validator": {
+      "version": "4.7.1",
+      "resolved": "https://registry.npmjs.org/openapi-examples-validator/-/openapi-examples-validator-4.7.1.tgz",
+      "integrity": "sha512-/OZZHhJkiaMQhkVfD0vFNOrRCBhkOL+X4/uhC55CJH0giuLUPQgFB5/iU26DCq1sZo2j9L70XYPcdHoM4PS+Xw==",
+      "dev": true,
+      "dependencies": {
+        "ajv": "^6.12.6",
+        "ajv-oai": "1.2.1",
+        "commander": "^6.2.1",
+        "errno": "^1.0.0",
+        "glob": "^7.2.0",
+        "json-pointer": "^0.6.2",
+        "json-schema-ref-parser": "^9.0.9",
+        "jsonpath-plus": "^6.0.1",
+        "lodash.clonedeep": "^4.5.0",
+        "lodash.flatmap": "^4.5.0",
+        "lodash.flatten": "^4.4.0",
+        "lodash.merge": "^4.6.2",
+        "yaml": "^1.10.2"
+      },
+      "bin": {
+        "openapi-examples-validator": "dist/cli.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/openapi-examples-validator/node_modules/ajv-oai": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/ajv-oai/-/ajv-oai-1.2.1.tgz",
+      "integrity": "sha512-gj7dnSdLyjWKid3uQI16u5wQNpkyqivjtCuvI4BWezeOzYTj5YHt4otH9GOBCaXY3FEbzQeWsp6C2qc18+BXDA==",
+      "dev": true,
+      "dependencies": {
+        "decimal.js": "^10.2.0"
+      },
+      "peerDependencies": {
+        "ajv": "6.x"
+      }
+    },
+    "node_modules/openapi-examples-validator/node_modules/commander": {
+      "version": "6.2.1",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-6.2.1.tgz",
+      "integrity": "sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==",
+      "dev": true,
+      "engines": {
+        "node": ">= 6"
+      }
+    },
+    "node_modules/openapi-examples-validator/node_modules/json-schema-ref-parser": {
+      "version": "9.0.9",
+      "resolved": "https://registry.npmjs.org/json-schema-ref-parser/-/json-schema-ref-parser-9.0.9.tgz",
+      "integrity": "sha512-qcP2lmGy+JUoQJ4DOQeLaZDqH9qSkeGCK3suKWxJXS82dg728Mn3j97azDMaOUmJAN4uCq91LdPx4K7E8F1a7Q==",
+      "deprecated": "Please switch to @apidevtools/json-schema-ref-parser",
+      "dev": true,
+      "dependencies": {
+        "@apidevtools/json-schema-ref-parser": "9.0.9"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/openapi-examples-validator/node_modules/jsonpath-plus": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-6.0.1.tgz",
+      "integrity": "sha512-EvGovdvau6FyLexFH2OeXfIITlgIbgZoAZe3usiySeaIDm5QS+A10DKNpaPBBqqRSZr2HN6HVNXxtwUAr2apEw==",
+      "dev": true,
+      "engines": {
+        "node": ">=10.0.0"
+      }
+    },
+    "node_modules/path-is-absolute": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+      "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/prr": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz",
+      "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=",
+      "dev": true
+    },
+    "node_modules/punycode": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
+      "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
+      "dev": true,
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/sprintf-js": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
+      "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=",
+      "dev": true
+    },
+    "node_modules/uri-js": {
+      "version": "4.4.1",
+      "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
+      "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==",
+      "dev": true,
+      "dependencies": {
+        "punycode": "^2.1.0"
+      }
+    },
+    "node_modules/wrappy": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
+      "dev": true
+    },
+    "node_modules/yaml": {
+      "version": "1.10.2",
+      "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz",
+      "integrity": "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==",
+      "dev": true,
+      "engines": {
+        "node": ">= 6"
+      }
+    }
+  },
+  "dependencies": {
+    "@apidevtools/json-schema-ref-parser": {
+      "version": "9.0.9",
+      "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.0.9.tgz",
+      "integrity": "sha512-GBD2Le9w2+lVFoc4vswGI/TjkNIZSVp7+9xPf+X3uidBfWnAeUWmquteSyt0+VCrhNMWj/FTABISQrD3Z/YA+w==",
+      "dev": true,
+      "requires": {
+        "@jsdevtools/ono": "^7.1.3",
+        "@types/json-schema": "^7.0.6",
+        "call-me-maybe": "^1.0.1",
+        "js-yaml": "^4.1.0"
+      },
+      "dependencies": {
+        "argparse": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
+          "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
+          "dev": true
+        },
+        "js-yaml": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
+          "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
+          "dev": true,
+          "requires": {
+            "argparse": "^2.0.1"
+          }
+        }
+      }
+    },
+    "@jsdevtools/ono": {
+      "version": "7.1.3",
+      "resolved": "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz",
+      "integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==",
+      "dev": true
+    },
+    "@types/json-schema": {
+      "version": "7.0.11",
+      "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz",
+      "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==",
+      "dev": true
+    },
+    "ajv": {
+      "version": "6.12.6",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
+      "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
+      "dev": true,
+      "requires": {
+        "fast-deep-equal": "^3.1.1",
+        "fast-json-stable-stringify": "^2.0.0",
+        "json-schema-traverse": "^0.4.1",
+        "uri-js": "^4.2.2"
+      }
+    },
+    "argparse": {
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
+      "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
+      "dev": true,
+      "requires": {
+        "sprintf-js": "~1.0.2"
+      }
+    },
+    "axios": {
+      "version": "0.21.4",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
+      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
+      "dev": true,
+      "requires": {
+        "follow-redirects": "^1.14.0"
+      }
+    },
+    "balanced-match": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
+      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
+      "dev": true
+    },
+    "brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
+      "requires": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "call-me-maybe": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/call-me-maybe/-/call-me-maybe-1.0.1.tgz",
+      "integrity": "sha1-JtII6onje1y95gJQoV8DHBak1ms=",
+      "dev": true
+    },
+    "concat-map": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
+      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
+      "dev": true
+    },
+    "decimal.js": {
+      "version": "10.2.1",
+      "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.2.1.tgz",
+      "integrity": "sha512-KaL7+6Fw6i5A2XSnsbhm/6B+NuEA7TZ4vqxnd5tXz9sbKtrN9Srj8ab4vKVdK8YAqZO9P1kg45Y6YLoduPf+kw==",
+      "dev": true
+    },
+    "errno": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/errno/-/errno-1.0.0.tgz",
+      "integrity": "sha512-3zV5mFS1E8/1bPxt/B0xxzI1snsg3uSCIh6Zo1qKg6iMw93hzPANk9oBFzSFBFrwuVoQuE3rLoouAUfwOAj1wQ==",
+      "dev": true,
+      "requires": {
+        "prr": "~1.0.1"
+      }
+    },
+    "esprima": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
+      "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
+      "dev": true
+    },
+    "fast-deep-equal": {
+      "version": "3.1.3",
+      "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
+      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
+      "dev": true
+    },
+    "fast-json-stable-stringify": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
+      "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==",
+      "dev": true
+    },
+    "follow-redirects": {
+      "version": "1.15.2",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
+      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
+      "dev": true
+    },
+    "foreach": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.6.tgz",
+      "integrity": "sha512-k6GAGDyqLe9JaebCsFCoudPPWfihKu8pylYXRlqP1J7ms39iPoTtk2fviNglIeQEwdh0bQeKJ01ZPyuyQvKzwg==",
+      "dev": true
+    },
+    "format-util": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/format-util/-/format-util-1.0.5.tgz",
+      "integrity": "sha512-varLbTj0e0yVyRpqQhuWV+8hlePAgaoFRhNFj50BNjEIrw1/DphHSObtqwskVCPWNgzwPoQrZAbfa/SBiicNeg==",
+      "dev": true
+    },
+    "fs.realpath": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
+      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
+      "dev": true
+    },
+    "glob": {
+      "version": "7.2.3",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+      "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
+      "dev": true,
+      "requires": {
+        "fs.realpath": "^1.0.0",
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "^3.1.1",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      }
+    },
+    "inflight": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
+      "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
+      "dev": true,
+      "requires": {
+        "once": "^1.3.0",
+        "wrappy": "1"
+      }
+    },
+    "inherits": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
+      "dev": true
+    },
+    "js-yaml": {
+      "version": "3.14.1",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz",
+      "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==",
+      "dev": true,
+      "requires": {
+        "argparse": "^1.0.7",
+        "esprima": "^4.0.0"
+      }
+    },
+    "json-pointer": {
+      "version": "0.6.2",
+      "resolved": "https://registry.npmjs.org/json-pointer/-/json-pointer-0.6.2.tgz",
+      "integrity": "sha512-vLWcKbOaXlO+jvRy4qNd+TI1QUPZzfJj1tpJ3vAXDych5XJf93ftpUKe5pKCrzyIIwgBJcOcCVRUfqQP25afBw==",
+      "dev": true,
+      "requires": {
+        "foreach": "^2.0.4"
+      }
+    },
+    "json-schema-traverse": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
+      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
+      "dev": true
+    },
+    "lodash.clonedeep": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
+      "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=",
+      "dev": true
+    },
+    "lodash.flatmap": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/lodash.flatmap/-/lodash.flatmap-4.5.0.tgz",
+      "integrity": "sha1-74y/QI9uSCaGYzRTBcaswLd4cC4=",
+      "dev": true
+    },
+    "lodash.flatten": {
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz",
+      "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=",
+      "dev": true
+    },
+    "lodash.merge": {
+      "version": "4.6.2",
+      "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
+      "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==",
+      "dev": true
+    },
+    "minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "dev": true,
+      "requires": {
+        "brace-expansion": "^1.1.7"
+      }
+    },
+    "once": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+      "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+      "dev": true,
+      "requires": {
+        "wrappy": "1"
+      }
+    },
+    "ono": {
+      "version": "4.0.11",
+      "resolved": "https://registry.npmjs.org/ono/-/ono-4.0.11.tgz",
+      "integrity": "sha512-jQ31cORBFE6td25deYeD80wxKBMj+zBmHTrVxnc6CKhx8gho6ipmWM5zj/oeoqioZ99yqBls9Z/9Nss7J26G2g==",
+      "dev": true,
+      "requires": {
+        "format-util": "^1.0.3"
+      }
+    },
+    "openapi-enforcer": {
+      "version": "1.13.1",
+      "resolved": "https://registry.npmjs.org/openapi-enforcer/-/openapi-enforcer-1.13.1.tgz",
+      "integrity": "sha512-NDDmyonl3bgxP+RabJsTPxn8EEza4Aet5zEocfX6nP9LL0J52aNFMYxNHoAuqoY3zQcZG2pnm3DMi2gYiRYbQg==",
+      "dev": true,
+      "requires": {
+        "axios": "^0.21.1",
+        "json-schema-ref-parser": "^6.1.0"
+      },
+      "dependencies": {
+        "json-schema-ref-parser": {
+          "version": "6.1.0",
+          "resolved": "https://registry.npmjs.org/json-schema-ref-parser/-/json-schema-ref-parser-6.1.0.tgz",
+          "integrity": "sha512-pXe9H1m6IgIpXmE5JSb8epilNTGsmTb2iPohAXpOdhqGFbQjNeHHsZxU+C8w6T81GZxSPFLeUoqDJmzxx5IGuw==",
+          "dev": true,
+          "requires": {
+            "call-me-maybe": "^1.0.1",
+            "js-yaml": "^3.12.1",
+            "ono": "^4.0.11"
+          }
+        }
+      }
+    },
+    "openapi-examples-validator": {
+      "version": "4.7.1",
+      "resolved": "https://registry.npmjs.org/openapi-examples-validator/-/openapi-examples-validator-4.7.1.tgz",
+      "integrity": "sha512-/OZZHhJkiaMQhkVfD0vFNOrRCBhkOL+X4/uhC55CJH0giuLUPQgFB5/iU26DCq1sZo2j9L70XYPcdHoM4PS+Xw==",
+      "dev": true,
+      "requires": {
+        "ajv": "^6.12.6",
+        "ajv-oai": "1.2.1",
+        "commander": "^6.2.1",
+        "errno": "^1.0.0",
+        "glob": "^7.2.0",
+        "json-pointer": "^0.6.2",
+        "json-schema-ref-parser": "^9.0.9",
+        "jsonpath-plus": "^6.0.1",
+        "lodash.clonedeep": "^4.5.0",
+        "lodash.flatmap": "^4.5.0",
+        "lodash.flatten": "^4.4.0",
+        "lodash.merge": "^4.6.2",
+        "yaml": "^1.10.2"
+      },
+      "dependencies": {
+        "ajv-oai": {
+          "version": "1.2.1",
+          "resolved": "https://registry.npmjs.org/ajv-oai/-/ajv-oai-1.2.1.tgz",
+          "integrity": "sha512-gj7dnSdLyjWKid3uQI16u5wQNpkyqivjtCuvI4BWezeOzYTj5YHt4otH9GOBCaXY3FEbzQeWsp6C2qc18+BXDA==",
+          "dev": true,
+          "requires": {
+            "decimal.js": "^10.2.0"
+          }
+        },
+        "commander": {
+          "version": "6.2.1",
+          "resolved": "https://registry.npmjs.org/commander/-/commander-6.2.1.tgz",
+          "integrity": "sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==",
+          "dev": true
+        },
+        "json-schema-ref-parser": {
+          "version": "9.0.9",
+          "resolved": "https://registry.npmjs.org/json-schema-ref-parser/-/json-schema-ref-parser-9.0.9.tgz",
+          "integrity": "sha512-qcP2lmGy+JUoQJ4DOQeLaZDqH9qSkeGCK3suKWxJXS82dg728Mn3j97azDMaOUmJAN4uCq91LdPx4K7E8F1a7Q==",
+          "dev": true,
+          "requires": {
+            "@apidevtools/json-schema-ref-parser": "9.0.9"
+          }
+        },
+        "jsonpath-plus": {
+          "version": "6.0.1",
+          "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-6.0.1.tgz",
+          "integrity": "sha512-EvGovdvau6FyLexFH2OeXfIITlgIbgZoAZe3usiySeaIDm5QS+A10DKNpaPBBqqRSZr2HN6HVNXxtwUAr2apEw==",
+          "dev": true
+        }
+      }
+    },
+    "path-is-absolute": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+      "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
+      "dev": true
+    },
+    "prr": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz",
+      "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=",
+      "dev": true
+    },
+    "punycode": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
+      "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
+      "dev": true
+    },
+    "sprintf-js": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
+      "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=",
+      "dev": true
+    },
+    "uri-js": {
+      "version": "4.4.1",
+      "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
+      "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==",
+      "dev": true,
+      "requires": {
+        "punycode": "^2.1.0"
+      }
+    },
+    "wrappy": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
+      "dev": true
+    },
+    "yaml": {
+      "version": "1.10.2",
+      "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz",
+      "integrity": "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==",
+      "dev": true
+    }
+  }
+}

package.json

@@ -0,0 +1,26 @@
+{
+  "name": "pihole-ftl",
+  "private": true,
+  "version": "1.0.0",
+  "description": "Source of the Pi-hole FTL daemon",
+  "main": "",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/pi-hole/FTL.git"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "EUPL-1.2",
+  "bugs": {
+    "url": "https://github.com/pi-hole/FTL/issues"
+  },
+  "scripts": {
+    "openapi-enforcer": "node test/api/openapi-enforcer.js",
+    "validate-examples": "openapi-examples-validator src/api/docs/content/specs/main.yaml",
+    "test": "npm run openapi-enforcer && npm run validate-examples"
+  },
+  "devDependencies": {
+    "openapi-enforcer": "^1.13.1",
+    "openapi-examples-validator": "^4.2.1"
+  }
+}

patch/civetweb.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+set -e
+
+patch -p1 < patch/civetweb/0001-add-pihole-mods.patch
+patch -p1 < patch/civetweb/0001-Add-NO_DLOPEN-option-to-civetweb-s-LUA-routines.patch
+patch -p1 < patch/civetweb/0001-Always-Kepler-syntax-for-Lua-server-pages.patch
+patch -p1 < patch/civetweb/0001-Add-FTL-URI-rewriting-changes-to-CivetWeb.patch
+patch -p1 < patch/civetweb/0001-Add-mbedTLS-debug-logging-hook.patch
+patch -p1 < patch/civetweb/0001-Register-CSRF-token-in-conn-request_info.patch
+patch -p1 < patch/civetweb/0001-Log-debug-messages-to-webserver.log-when-debug.webse.patch
+patch -p1 < patch/civetweb/0001-Allow-extended-ASCII-characters-in-URIs.patch
+
+echo "ALL PATCHES APPLIED OKAY"

patch/civetweb/0001-Add-FTL-URI-rewriting-changes-to-CivetWeb.patch

@@ -0,0 +1,41 @@
+From 44a0a3a27731bb8577f39743eb034c024be48df2 Mon Sep 17 00:00:00 2001
+From: DL6ER <dl6er@dl6er.de>
+Date: Thu, 25 May 2023 18:02:13 +0200
+Subject: [PATCH] Add FTL URI rewriting changes to CivetWeb
+
+Signed-off-by: DL6ER <dl6er@dl6er.de>
+---
+ src/webserver/civetweb/civetweb.c | 2 ++
+ src/webserver/civetweb/civetweb.h | 3 +++
+ 2 files changed, 5 insertions(+)
+
+diff --git a/src/webserver/civetweb/civetweb.c b/src/webserver/civetweb/civetweb.c
+index 0d293f1f..44f6cf3d 100644
+--- a/src/webserver/civetweb/civetweb.c
++++ b/src/webserver/civetweb/civetweb.c
+@@ -7754,6 +7754,8 @@ interpret_uri(struct mg_connection *conn, /* in/out: request (must be valid) */
+ 	mg_snprintf(
+ 	    conn, &truncated, filename, filename_buf_len - 1, "%s%s", root, uri);
+ 
++	FTL_rewrite_pattern(filename, filename_buf_len - 1, root, uri);
++
+ 	if (truncated) {
+ 		goto interpret_cleanup;
+ 	}
+diff --git a/src/webserver/civetweb/civetweb.h b/src/webserver/civetweb/civetweb.h
+index e71dfedc..2ad76693 100644
+--- a/src/webserver/civetweb/civetweb.h
++++ b/src/webserver/civetweb/civetweb.h
+@@ -935,6 +935,9 @@ int my_send_http_error_headers(struct mg_connection *conn,
+                                int status, const char* mime_type,
+                                long long content_length);
+ 
++void FTL_rewrite_pattern(char *filename, size_t filename_buf_len,
++                         const char *root, const char *uri);
++
+ // Buffer used for additional "Set-Cookie" headers
+ #define PIHOLE_HEADERS_MAXLEN 1024
+ extern char pi_hole_extra_headers[PIHOLE_HEADERS_MAXLEN];
+-- 
+2.34.1
+

patch/civetweb/0001-Add-NO_DLOPEN-option-to-civetweb-s-LUA-routines.patch

@@ -0,0 +1,35 @@
+From 1b81285fed48df6939d4b2569bba9e572f4c1137 Mon Sep 17 00:00:00 2001
+From: DL6ER <dl6er@dl6er.de>
+Date: Fri, 13 Jan 2023 21:37:31 +0100
+Subject: [PATCH] Add NO_DLOPEN option to civetweb's LUA routines
+
+Signed-off-by: DL6ER <dl6er@dl6er.de>
+---
+ src/webserver/civetweb/mod_lua.inl | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/src/webserver/civetweb/mod_lua.inl b/src/webserver/civetweb/mod_lua.inl
+index 5cc94318..59c4f2b3 100644
+--- a/src/webserver/civetweb/mod_lua.inl
++++ b/src/webserver/civetweb/mod_lua.inl
+@@ -3634,7 +3634,7 @@ lua_init_optional_libraries(void)
+ 	lua_shared_init();
+ 
+ /* UUID library */
+-#if !defined(_WIN32)
++#if !defined(_WIN32) && !defined(NO_DLOPEN)
+ 	lib_handle_uuid = dlopen("libuuid.so", RTLD_LAZY);
+ 	pf_uuid_generate.p =
+ 	    (lib_handle_uuid ? dlsym(lib_handle_uuid, "uuid_generate") : 0);
+@@ -3648,7 +3648,7 @@ static void
+ lua_exit_optional_libraries(void)
+ {
+ /* UUID library */
+-#if !defined(_WIN32)
++#if !defined(_WIN32) && !defined(NO_DLOPEN)
+ 	if (lib_handle_uuid) {
+ 		dlclose(lib_handle_uuid);
+ 	}
+-- 
+2.34.1
+

patch/civetweb/0001-Add-mbedTLS-debug-logging-hook.patch

@@ -0,0 +1,44 @@
+From f785e181f8b43fa9f77bf7dcc6711f16206c9e89 Mon Sep 17 00:00:00 2001
+From: DL6ER <dl6er@dl6er.de>
+Date: Thu, 25 May 2023 18:26:45 +0200
+Subject: [PATCH] Add mbedTLS debug logging hook
+
+Signed-off-by: DL6ER <dl6er@dl6er.de>
+---
+ src/webserver/civetweb/civetweb.h      | 4 ++++
+ src/webserver/civetweb/mod_mbedtls.inl | 4 ++++
+ 2 files changed, 8 insertions(+)
+
+diff --git a/src/webserver/civetweb/civetweb.h b/src/webserver/civetweb/civetweb.h
+index 2ad76693..52724199 100644
+--- a/src/webserver/civetweb/civetweb.h
++++ b/src/webserver/civetweb/civetweb.h
+@@ -938,6 +938,10 @@ int my_send_http_error_headers(struct mg_connection *conn,
+ void FTL_rewrite_pattern(char *filename, size_t filename_buf_len,
+                          const char *root, const char *uri);
+ 
++#define MG_CONFIG_MBEDTLS_DEBUG 3
++void FTL_mbed_debug(void *user_param, int level, const char *file,
++                    int line, const char *message);
++
+ // Buffer used for additional "Set-Cookie" headers
+ #define PIHOLE_HEADERS_MAXLEN 1024
+ extern char pi_hole_extra_headers[PIHOLE_HEADERS_MAXLEN];
+diff --git a/src/webserver/civetweb/mod_mbedtls.inl b/src/webserver/civetweb/mod_mbedtls.inl
+index e72685f4..00b9280a 100644
+--- a/src/webserver/civetweb/mod_mbedtls.inl
++++ b/src/webserver/civetweb/mod_mbedtls.inl
+@@ -83,6 +83,10 @@ mbed_sslctx_init(SSL_CTX *ctx, const char *crt)
+ 	mbedtls_ssl_conf_dbg(conf, mbed_debug, (void *)ctx);
+ #endif
+ 
++	/****************** Pi-hole change ******************/
++	mbedtls_ssl_conf_dbg(conf, FTL_mbed_debug, NULL);
++	/****************************************************/
++
+ #ifdef MBEDTLS_SSL_PROTO_TLS1_3
+ 	psa_status_t status = psa_crypto_init();
+ 	if (status != PSA_SUCCESS) {
+-- 
+2.34.1
+

patch/civetweb/0001-Allow-extended-ASCII-characters-in-URIs.patch

@@ -0,0 +1,35 @@
+From ebb27741b10ed2eac51ac356708800ae96cdd17a Mon Sep 17 00:00:00 2001
+From: DL6ER <dl6er@dl6er.de>
+Date: Tue, 31 Oct 2023 08:35:31 +0100
+Subject: [PATCH] Allow extended ASCII characters in URIs
+
+Signed-off-by: DL6ER <dl6er@dl6er.de>
+---
+ src/webserver/civetweb/civetweb.c | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/src/webserver/civetweb/civetweb.c b/src/webserver/civetweb/civetweb.c
+index 9b0c6308..5320c4d4 100644
+--- a/src/webserver/civetweb/civetweb.c
++++ b/src/webserver/civetweb/civetweb.c
+@@ -10734,7 +10734,7 @@ skip_to_end_of_word_and_terminate(char **ppw, int eol)
+ {
+ 	/* Forward until a space is found - use isgraph here */
+ 	/* See http://www.cplusplus.com/reference/cctype/ */
+-	while (isgraph((unsigned char)**ppw)) {
++	while ((unsigned char)**ppw > 127 || isgraph((unsigned char)**ppw)) {
+ 		(*ppw)++;
+ 	}
+ 
+@@ -18473,7 +18473,7 @@ get_uri_type(const char *uri)
+ 	 * and % encoded symbols.
+ 	 */
+ 	for (i = 0; uri[i] != 0; i++) {
+-		if (uri[i] < 33) {
++		if ((unsigned char)uri[i] < 33) {
+ 			/* control characters and spaces are invalid */
+ 			return 0;
+ 		}
+-- 
+2.34.1
+

patch/civetweb/0001-Always-Kepler-syntax-for-Lua-server-pages.patch

@@ -0,0 +1,31 @@
+From 19efd3e2e10858b549404e6cae97c20337aafb0b Mon Sep 17 00:00:00 2001
+From: DL6ER <dl6er@dl6er.de>
+Date: Mon, 22 May 2023 19:11:44 +0200
+Subject: [PATCH] Always Kepler syntax for Lua server pages
+
+Signed-off-by: DL6ER <dl6er@dl6er.de>
+---
+ src/webserver/civetweb/mod_lua.inl | 7 +++----
+ 1 file changed, 3 insertions(+), 4 deletions(-)
+
+diff --git a/src/webserver/civetweb/mod_lua.inl b/src/webserver/civetweb/mod_lua.inl
+index 26f281ee..e9a13835 100644
+--- a/src/webserver/civetweb/mod_lua.inl
++++ b/src/webserver/civetweb/mod_lua.inl
+@@ -3208,10 +3208,9 @@ handle_lsp_request(struct mg_connection *conn,
+ 	 * "<?" which means "classic CivetWeb Syntax".
+ 	 *
+ 	 */
+-	run_lsp = run_lsp_civetweb;
+-	if ((addr[0] == '<') && (addr[1] != '?')) {
+-		run_lsp = run_lsp_kepler;
+-	}
++
++	// Pi-hole change: Always use Kepler syntax, ignore rules above
++	run_lsp = run_lsp_kepler;
+ 
+ 	/* We're not sending HTTP headers here, Lua page must do it. */
+ 	error =
+-- 
+2.34.1
+

patch/civetweb/0001-Log-debug-messages-to-webserver.log-when-debug.webse.patch

@@ -0,0 +1,54 @@
+From 445aed4fd5ef53a52ac974aa5123ba56b11d8a1f Mon Sep 17 00:00:00 2001
+From: DL6ER <dl6er@dl6er.de>
+Date: Sat, 14 Oct 2023 15:39:21 +0200
+Subject: [PATCH] Log debug messages to webserver.log when debug.webserver is
+ true
+
+Signed-off-by: DL6ER <dl6er@dl6er.de>
+---
+ src/webserver/civetweb/civetweb.c      | 5 +++--
+ src/webserver/civetweb/mod_mbedtls.inl | 4 ++--
+ 2 files changed, 5 insertions(+), 4 deletions(-)
+
+diff --git a/src/webserver/civetweb/civetweb.c b/src/webserver/civetweb/civetweb.c
+index 3df8eab9..9b0c6308 100644
+--- a/src/webserver/civetweb/civetweb.c
++++ b/src/webserver/civetweb/civetweb.c
+@@ -239,9 +239,10 @@ static void DEBUG_TRACE_FUNC(const char *func,
+ #endif
+ 
+ #else
++#include "log.h"
+ #define DEBUG_TRACE(fmt, ...)                                                  \
+-	do {                                                                       \
+-	} while (0)
++	if(debug_flags[DEBUG_WEBSERVER]) {\
++		log_web("DEBUG: " fmt " (%s:%d)", ##__VA_ARGS__, short_path(__FILE__), __LINE__); }
+ #endif /* DEBUG */
+ #endif /* DEBUG_TRACE */
+ 
+diff --git a/src/webserver/civetweb/mod_mbedtls.inl b/src/webserver/civetweb/mod_mbedtls.inl
+index 00b9280a..6a450ba3 100644
+--- a/src/webserver/civetweb/mod_mbedtls.inl
++++ b/src/webserver/civetweb/mod_mbedtls.inl
+@@ -213,7 +213,7 @@ mbed_ssl_accept(mbedtls_ssl_context **ssl,
+ 		return -1;
+ 	}
+ 
+-	DEBUG_TRACE("TLS connection %p accepted, state: %d", ssl, (*ssl)->state);
++	DEBUG_TRACE("TLS connection %p accepted, state: %d", ssl, (*ssl)->MBEDTLS_PRIVATE(state));
+ 	return 0;
+ }
+ 
+@@ -239,7 +239,7 @@ mbed_ssl_handshake(mbedtls_ssl_context *ssl)
+ 		}
+ 	}
+ 
+-	DEBUG_TRACE("TLS handshake rc: %d, state: %d", rc, ssl->state);
++	DEBUG_TRACE("TLS handshake rc: %d, state: %d", rc, ssl->MBEDTLS_PRIVATE(state));
+ 	return rc;
+ }
+ 
+-- 
+2.34.1
+

patch/civetweb/0001-Register-CSRF-token-in-conn-request_info.patch

@@ -0,0 +1,58 @@
+From 4890e1c2586d4a74a3925ede8cfeb1805672a42e Mon Sep 17 00:00:00 2001
+From: DL6ER <dl6er@dl6er.de>
+Date: Sat, 3 Jun 2023 20:52:02 +0200
+Subject: [PATCH] Register CSRF token and is_authenticated boolean in conn->request_info
+
+Signed-off-by: DL6ER <dl6er@dl6er.de>
+---
+ src/webserver/civetweb/civetweb.c  | 3 +++
+ src/webserver/civetweb/civetweb.h  | 2 ++
+ src/webserver/civetweb/mod_lua.inl | 3 +++
+ 3 files changed, 8 insertions(+)
+
+diff --git a/src/webserver/civetweb/civetweb.c b/src/webserver/civetweb/civetweb.c
+index 233b342a..f44b17ba 100644
+--- a/src/webserver/civetweb/civetweb.c
++++ b/src/webserver/civetweb/civetweb.c
+@@ -17760,6 +17760,9 @@ reset_per_request_attributes(struct mg_connection *conn)
+ 	}
+ 	conn->request_info.local_uri = NULL;
+ 
++	/* Pi-hole addition */
++	memset(conn->request_info.csrf_token, 0, sizeof(conn->request_info.csrf_token));
++	reg_boolean(L, "is_authenticated", conn->request_info.is_authenticated != 0);
++
+ #if defined(USE_SERVER_STATS)
+ 	conn->processing_time = 0;
+ #endif
+diff --git a/src/webserver/civetweb/civetweb.h b/src/webserver/civetweb/civetweb.h
+index 5b3d596b..291ef683 100644
+--- a/src/webserver/civetweb/civetweb.h
++++ b/src/webserver/civetweb/civetweb.h
+@@ -183,6 +183,8 @@ struct mg_request_info {
+ 
+ 	const char *acceptedWebSocketSubprotocol; /* websocket subprotocol,
+ 	                                           * accepted during handshake */
++	/* Pi-hole modification */
++	char csrf_token[32];
++	int is_authenticated;
+ };
+ 
+ 
+diff --git a/src/webserver/civetweb/mod_lua.inl b/src/webserver/civetweb/mod_lua.inl
+index e9a13835..92066b3f 100644
+--- a/src/webserver/civetweb/mod_lua.inl
++++ b/src/webserver/civetweb/mod_lua.inl
+@@ -2603,6 +2603,9 @@ prepare_lua_request_info_inner(const struct mg_connection *conn, lua_State *L)
+ 		reg_string(L, "finger", conn->request_info.client_cert->finger);
+ 		lua_rawset(L, -3);
+ 	}
++
++	/* Pi-hole addition */
++	reg_string(L, "csrf_token", conn->request_info.csrf_token);
+ }
+ 
+ 
+-- 
+2.34.1
+

patch/civetweb/0001-add-pihole-mods.patch

@@ -0,0 +1,103 @@
+From f59235a1973d13d1271b50546c28cf4d9d2f921a Mon Sep 17 00:00:00 2001
+From: DL6ER <dl6er@dl6er.de>
+Date: Sat, 31 Dec 2022 07:08:23 +0100
+Subject: [PATCH] Add Pi-hole specific method to send HTTP headers
+
+Signed-off-by: DL6ER <dl6er@dl6er.de>
+---
+ src/webserver/civetweb/civetweb.c | 53 +++++++++++++++++++++++++++++++++++++++++
+ src/webserver/civetweb/civetweb.h | 11 +++++++++
+ 2 files changed, 64 insertions(+)
+
+diff --git a/src/webserver/civetweb/civetweb.c b/src/webserver/civetweb/civetweb.c
+index 81f642be..ed360a76 100644
+--- a/src/webserver/civetweb/civetweb.c
++++ b/src/webserver/civetweb/civetweb.c
+@@ -4135,6 +4135,14 @@ send_additional_header(struct mg_connection *conn)
+ 	if (header && header[0]) {
+ 		mg_response_header_add_lines(conn, header);
+ 	}
++
++	/*************** Pi-hole modification ****************/
++	if (pi_hole_extra_headers[0] != '\0') {
++		mg_response_header_add_lines(conn, pi_hole_extra_headers);
++		// Invalidate extra headers after having sent them to avoid repetitions
++		pi_hole_extra_headers[0] = '\0';
++	}
++	/*****************************************************/
+ }
+ 
+ 
+@@ -4530,6 +4538,48 @@ mg_send_http_error_impl(struct mg_connection *conn,
+ }
+ 
+ 
++/************************************** Pi-hole method **************************************/
++CIVETWEB_API int
++my_send_http_error_headers(struct mg_connection *conn,
++                           int status, const char* mime_type,
++                           long long content_length)
++{
++	if ((mime_type == NULL) || (*mime_type == 0)) {
++		/* No content type defined: default to text/html */
++		mime_type = "text/html";
++	}
++
++	mg_response_header_start(conn, status);
++	send_no_cache_header(conn);
++	send_additional_header(conn);
++	mg_response_header_add(conn, "Content-Type", mime_type, -1);
++	if (content_length < 0) {
++		/* Size not known. Use chunked encoding (HTTP/1.x) */
++		if (conn->protocol_type == PROTOCOL_TYPE_HTTP1) {
++			/* Only HTTP/1.x defines "chunked" encoding, HTTP/2 does not*/
++			mg_response_header_add(conn, "Transfer-Encoding", "chunked", -1);
++		}
++	} else {
++		char len[32];
++		int trunc = 0;
++		mg_snprintf(conn,
++		            &trunc,
++		            len,
++		            sizeof(len),
++		            "%" UINT64_FMT,
++		            (uint64_t)content_length);
++		if (!trunc) {
++			/* Since 32 bytes is enough to hold any 64 bit decimal number,
++			 * !trunc is always true */
++			mg_response_header_add(conn, "Content-Length", len, -1);
++		}
++	}
++	mg_response_header_send(conn);
++
++	return 0;
++}
++/********************************************************************************************/
++
+ CIVETWEB_API int
+ mg_send_http_error(struct mg_connection *conn, int status, const char *fmt, ...)
+ {
+diff --git a/src/webserver/civetweb/civetweb.h b/src/webserver/civetweb/civetweb.h
+index 7ea45fb2..f879ff3e 100644
+--- a/src/webserver/civetweb/civetweb.h
++++ b/src/webserver/civetweb/civetweb.h
+@@ -963,6 +964,16 @@ CIVETWEB_API int mg_send_http_error(struct mg_connection *conn,
+                                     PRINTF_FORMAT_STRING(const char *fmt),
+                                     ...) PRINTF_ARGS(3, 4);
+ 
++/************************************** Pi-hole method **************************************/
++int my_send_http_error_headers(struct mg_connection *conn,
++                               int status, const char* mime_type,
++                               long long content_length);
++
++// Buffer used for additional "Set-Cookie" headers
++#define PIHOLE_HEADERS_MAXLEN 1024
++extern char pi_hole_extra_headers[PIHOLE_HEADERS_MAXLEN];
++/********************************************************************************************/
++
+ 
+ /* Send "HTTP 200 OK" response header.
+  * After calling this function, use mg_write or mg_send_chunk to send the
+-- 
+2.34.1
+

patch/lua.sh

@@ -2,3 +2,5 @@
 set -e
 
 patch -p1 < patch/lua/0001-add-pihole-library.patch
+
+echo "ALL PATCHES APPLIED OKAY"

patch/sqlite3.sh

@@ -3,3 +3,5 @@ set -e
 
 patch -p1 < patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch
 patch -p1 < patch/sqlite3/0002-make-sqlite3ErrName-public.patch
+
+echo "ALL PATCHES APPLIED OKAY"

patch/sqlite3/0001-print-FTL-version-in-interactive-shell.patch

@@ -25,6 +25,6 @@ index 6280ebf6..a5e82f70 100644
        char *zHistory;
        int nHistory;
 +      print_FTL_version();
-       printf(
-         "SQLite version %s %.19s\n" /*extra-version-info*/
-         "Enter \".help\" for usage hints.\n",
+ #if CIO_WIN_WC_XLATE
+ # define SHELL_CIO_CHAR_SET (stdout_is_console? " (UTF-16 console I/O)" : "")
+ #else

src/CMakeLists.txt

@@ -24,13 +24,20 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR})
 # SQLITE_DEFAULT_MEMSTATUS=0: This setting causes the sqlite3_status() interfaces that track memory usage to be disabled. This helps the sqlite3_malloc() routines run much faster, and since SQLite uses sqlite3_malloc() internally, this helps to make the entire library faster.
 # SQLITE_OMIT_DEPRECATED: Omitting deprecated interfaces and features will not help SQLite to run any faster. It will reduce the library footprint, however. And it is the right thing to do.
 # SQLITE_OMIT_PROGRESS_CALLBACK: The progress handler callback counter must be checked in the inner loop of the bytecode engine. By omitting this interface, a single conditional is removed from the inner loop of the bytecode engine, helping SQL statements to run slightly faster.
+# SQLITE_OMIT_SHARED_CACHE: This option builds SQLite without support for shared cache mode. The sqlite3_enable_shared_cache() is omitted along with a fair amount of logic within the B-Tree subsystem associated with shared cache management. This compile-time option is recommended most applications as it results in improved performance and reduced library footprint.
 # SQLITE_DEFAULT_FOREIGN_KEYS=1: This macro determines whether enforcement of foreign key constraints is enabled or disabled by default for new database connections.
 # SQLITE_DQS=0: This setting disables the double-quoted string literal misfeature.
 # SQLITE_ENABLE_DBPAGE_VTAB: Enables the SQLITE_DBPAGE virtual table. Warning: writing to the SQLITE_DBPAGE virtual table can very easily cause unrecoverably database corruption.
-# SQLITE_OMIT_DESERIALIZE: This option causes the the sqlite3_serialize() and sqlite3_deserialize() interfaces to be omitted from the build (was the default before 3.36.0)
+# SQLITE_TEMP_STORE=2: Store temporary tables in memory for reduced IO and higher performance (can be overwritten by the user at runtime).
 # HAVE_READLINE: Enable readline support to allow easy editing, history and auto-completion
 # SQLITE_DEFAULT_CACHE_SIZE=-16384: Allow up to 16 MiB of cache to be used by SQLite3 (default is 2000 kiB)
-set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_OMIT_DESERIALIZE -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=-16384")
+# SQLITE_DEFAULT_SYNCHRONOUS=1: Use normal synchronous mode (default is 2)
+# SQLITE_LIKE_DOESNT_MATCH_BLOBS: This option causes the LIKE operator to only match BLOB values against BLOB values and TEXT values against TEXT values. This compile-time option makes SQLite run more efficiently when processing queries that use the LIKE operator.
+# HAVE_MALLOC_USABLE_SIZE: This option causes SQLite to try to use the malloc_usable_size() function to obtain the actual size of memory allocations from the underlying malloc() system interface. Applications are encouraged to use HAVE_MALLOC_USABLE_SIZE whenever possible.
+# HAVE_FDATASYNC: This option causes SQLite to try to use the fdatasync() system call to sync the database file to disk when committing a transaction. Syncing using fdatasync() is faster than syncing using fsync() as fdatasync() does not wait for the file metadata to be written to disk.
+# SQLITE_DEFAULT_WORKER_THREADS=4: This option sets the default number of worker threads to use when doing parallel sorting and indexing. The default is 0 which means to use a single thread. The default for SQLITE_MAX_WORKER_THREADS is 8.
+# SQLITE_MAX_PREPARE_RETRY=200: This option sets the maximum number of automatic re-preparation attempts that can occur after encountering a schema change. This can be caused by running ANALYZE which is done periodically by FTL.
+set(SQLITE_DEFINES "-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_DEFAULT_FOREIGN_KEYS=1 -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_TEMP_STORE=2 -DHAVE_READLINE -DSQLITE_DEFAULT_CACHE_SIZE=16384 -DSQLITE_DEFAULT_SYNCHRONOUS=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DHAVE_MALLOC_USABLE_SIZE -DHAVE_FDATASYNC -DSQLITE_DEFAULT_WORKER_THREADS=4 -DSQLITE_MAX_PREPARE_RETRY=200")
 
 # Code hardening and debugging improvements
 # -fstack-protector-strong: The program will be resistant to having its stack overflowed
@@ -55,43 +62,127 @@ set(DEBUG_FLAGS "-rdynamic -fno-omit-frame-pointer")
 set(WARN_FLAGS "-Wall -Wextra -Wno-unused-parameter")
 
 # Extra warning flags we apply only to the FTL part of the code (used not for foreign code such as dnsmasq and SQLite3)
-# -Werror: Halt on any warnings, useful for enforcing clean code without any warnings (we use it only for our code part)
-# -Waddress: Warn about suspicious uses of memory addresses
-# -Wlogical-op: Warn about suspicious uses of logical operators in expressions
-# -Wmissing-field-initializers: Warn if a structure's initializer has some fields missing
-# -Woverlength-strings: Warn about string constants that are longer than the "minimum maximum length specified in the C standard
-# -Wformat: Check calls to printf and scanf, etc., to make sure that the arguments supplied have types appropriate to the format string specified, and that the conversions specified in the format string make sense.
-# -Wformat-nonliteral: If -Wformat is specified, also warn if the format string is not a string literal and so cannot be checked, unless the format function takes its format arguments as a va_list.
-# -Wuninitialized: Warn if an automatic variable is used without first being initialized
-# -Wswitch-enum: Warn whenever a switch statement has an index of enumerated type and lacks a case for one or more of the named codes of that enumeration.
-# -Wshadow: Warn whenever a local variable or type declaration shadows another variable, parameter, type, class member, or whenever a built-in function is shadowed.
-# -Wfloat-equal: Warn if floating-point values are used in equality comparisons
-# -Wpointer-arith: Warn about anything that depends on the "size of" a function type or of "void".  GNU C assigns these types a size of 1
-# -Wundef: Warn if an undefined identifier is evaluated in an "#if" directive
-# -Wbad-function-cast: Warn when a function call is cast to a non-matching type
-# -Wwrite-strings: When compiling C, give string constants the type "const char[length]" so that copying the address of one into a non-"const" "char *" pointer produces a warning
-# -Wparentheses: Warn if parentheses are omitted in certain contexts, such as when there is an assignment in a context where a truth value is expected, or when operators are nested whose precedence people often get confused about
-# -Wlogical-op: Warn about suspicious uses of logical operators in expressions
-# -Wstrict-prototypes: Warn if a function is declared or defined without specifying the argument types
-# -Wmissing-prototypes: Warn if a global function is defined without a previous prototype declaration
-# -Wredundant-decls: Warn if anything is declared more than once in the same scope
-# -Winline: Warn if a function that is declared as inline cannot be inlined
+set(EXTRAWARN_GCC6 "-Werror \
+                    -Waddress \
+                    -Wlogical-op \
+                    -Wmissing-field-initializers \
+                    -Woverlength-strings \
+                    -Wformat=2 \
+                    -Wformat-signedness \
+                    -Wuninitialized \
+                    -Wnull-dereference \
+                    -Wshift-overflow=2 \
+                    -Wunused-const-variable=2 \
+                    -Wstrict-aliasing \
+                    -Warray-bounds=2 \
+                    -Wno-aggressive-loop-optimizations \
+                    -Wswitch-enum \
+                    -Wshadow \
+                    -Wfloat-equal \
+                    -Wbad-function-cast \
+                    -Wwrite-strings \
+                    -Wparentheses \
+                    -Wlogical-op \
+                    -Wstrict-prototypes \
+                    -Wmissing-prototypes \
+                    -Wredundant-decls \
+                    -Wmissing-field-initializers \
+                    -Wnormalized=nfkc \
+                    -Woverride-init \
+                    -Wpacked \
+                    -Winline \
+                    -Wpacked \
+                    -Wredundant-decls \
+                    -Wnested-externs \
+                    -Wvla \
+                    -Wvector-operation-performance \
+                    -Wvolatile-register-var \
+                    -Wdisabled-optimization \
+                    -Wpointer-sign \
+                    -Wstack-protector \
+                    -Woverlength-strings")
+
+# Extra warnings flags available only in GCC 7 and higher
+if(CMAKE_C_COMPILER_VERSION VERSION_EQUAL 7 OR CMAKE_C_COMPILER_VERSION VERSION_GREATER 7)
+    set(EXTRAWARN_GCC7 "-Wformat-overflow=2 \
+                        -Wformat-truncation=2 \
+                        -Wstringop-overflow=4 \
+                        -Walloc-zero \
+                        -Wint-in-bool-context")
+else()
+    set(EXTRAWARN_GCC7 "")
+endif()
+
+# Extra warnings flags available only in GCC 8 and higher
 if(CMAKE_C_COMPILER_VERSION VERSION_EQUAL 8 OR CMAKE_C_COMPILER_VERSION VERSION_GREATER 8)
-    # -Wduplicated-cond: Warn about duplicated conditions in an if-else-if chain
-    # -Wduplicated-branches: Warn when an if-else has identical branches
-    # -Wcast-align=strict: Warn whenever a pointer is cast such that the required alignment of the target is increased. For example, warn if a "char *" is cast to an "int *" regardless of the target machine.
-    # -Wlogical-not-parentheses: Warn about logical not used on the left hand side operand of a comparison
-    set(EXTRAWARN_GCC8 "-Wduplicated-cond -Wduplicated-branches -Wcast-align=strict -Wlogical-not-parentheses -Wsuggest-attribute=pure -Wsuggest-attribute=const -Wsuggest-attribute=malloc -Wsuggest-attribute=format -Wsuggest-attribute=cold")
+    set(EXTRAWARN_GCC8 "-Wduplicated-cond \
+                        -Wduplicated-branches \
+                        -Wcast-align=strict \
+                        -Wlogical-not-parentheses \
+                        -Wmultistatement-macros \
+                        -Wmissing-attributes \
+                        -Wsuggest-attribute=pure \
+                        -Wsuggest-attribute=const \
+                        -Wsuggest-attribute=malloc \
+                        -Wsuggest-attribute=format \
+                        -Wsuggest-attribute=cold")
 else()
     set(EXTRAWARN_GCC8 "")
 endif()
-set(EXTRAWARN "-Werror -Waddress -Wlogical-op -Wmissing-field-initializers -Woverlength-strings -Wformat -Wformat-nonliteral -Wuninitialized -Wswitch-enum -Wshadow -Wfloat-equal -Wbad-function-cast -Wwrite-strings -Wparentheses -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes -Wredundant-decls -Winline ${EXTRAWARN_GCC8}")
+
+# Extra warnings flags available only in GCC 9 and higher
+# The only new warning -Wabsolute-value is implied by -Wextra
+
+# Extra warnings flags available only in GCC 10 and higher
+# The new option -Wstring-compare is implied by -Wextra
+# The new option -Wzero-length-bounds is implied by -Warray-bounds
+
+# Extra warnings flags available only in GCC 11 and higher
+# All new options are implied by either -Wall or -Wextra \
+
+# Extra warnings flags available only in GCC 12 and higher
+if(CMAKE_C_COMPILER_VERSION VERSION_EQUAL 12 OR CMAKE_C_COMPILER_VERSION VERSION_GREATER 12)
+set(EXTRAWARN_GCC12 "-Wbidi-chars \
+                     -Warray-compare")
+else()
+set(EXTRAWARN_GCC12 "")
+endif()
+
+# Extra warnings flags available only in GCC 13 and higher
+if(CMAKE_C_COMPILER_VERSION VERSION_EQUAL 13 OR CMAKE_C_COMPILER_VERSION VERSION_GREATER 13)
+set(EXTRAWARN_GCC13 "-Wenum-int-mismatch")
+else()
+set(EXTRAWARN_GCC13 "")
+endif()
+
+set(EXTRAWARN "${EXTRAWARN_GCC6} \
+               ${EXTRAWARN_GCC7} \
+               ${EXTRAWARN_GCC8} \
+               ${EXTRAWARN_GCC12} \
+               ${EXTRAWARN_GCC13}")
+
+# Remove extra spaces from EXTRAWARN
+string(REGEX REPLACE " +" " " EXTRAWARN "${EXTRAWARN}")
+
+# Separate EXTRAWARN into a list of arguments
 separate_arguments(EXTRAWARN)
 
-# Do we want to compile a statically linked musl executable?
-if(STATIC STREQUAL "true")
+# -Wxor-used-as-pow
+
+# Do we want to compile a statically linked executable?
+if(DEFINED ENV{STATIC})
+    if($ENV{STATIC} STREQUAL "true")
+        set(STATIC true)
+    else()
+        set(STATIC false)
+    endif()
+endif()
+if(STATIC)
+    message(STATUS "Compiling statically linked executable")
     SET(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
     SET(BUILD_SHARED_LIBS OFF)
+else()
+    message(STATUS "Compiling dynamically linked executable")
 endif()
 # -pie -fPIE: (Dynamic) position independent executable
 set(HARDENING_FLAGS "${HARDENING_FLAGS} -pie -fPIE")
@@ -110,8 +201,6 @@ set(sources
         args.h
         capabilities.c
         capabilities.h
-        config.c
-        config.h
         daemon.c
         daemon.h
         datastructure.c
@@ -132,6 +221,7 @@ set(sources
         log.h
         main.c
         main.h
+        metrics.h
         overTime.c
         overTime.h
         procps.c
@@ -140,8 +230,6 @@ set(sources
         regex_r.h
         resolve.c
         resolve.h
-        setupVars.c
-        setupVars.h
         shmem.c
         shmem.h
         signals.c
@@ -172,15 +260,23 @@ add_dependencies(FTL gen_version)
 add_executable(pihole-FTL
         $<TARGET_OBJECTS:FTL>
         $<TARGET_OBJECTS:api>
+        $<TARGET_OBJECTS:api_docs>
+        $<TARGET_OBJECTS:webserver>
+        $<TARGET_OBJECTS:civetweb>
+        $<TARGET_OBJECTS:cJSON>
+        $<TARGET_OBJECTS:miniz>
+        $<TARGET_OBJECTS:zip>
         $<TARGET_OBJECTS:database>
         $<TARGET_OBJECTS:dnsmasq>
         $<TARGET_OBJECTS:sqlite3>
         $<TARGET_OBJECTS:lua>
         $<TARGET_OBJECTS:tre-regex>
         $<TARGET_OBJECTS:syscalls>
+        $<TARGET_OBJECTS:tomlc99>
+        $<TARGET_OBJECTS:config>
         $<TARGET_OBJECTS:tools>
         )
-if(STATIC STREQUAL "true")
+if(STATIC)
     set_target_properties(pihole-FTL PROPERTIES LINK_SEARCH_START_STATIC ON)
     set_target_properties(pihole-FTL PROPERTIES LINK_SEARCH_END_STATIC ON)
     target_link_libraries(pihole-FTL -static-libgcc -static -pie)
@@ -193,12 +289,15 @@ set(THREADS_PREFER_PTHREAD_FLAG TRUE)
 find_package(Threads REQUIRED)
 
 # for DNSSEC we need the nettle (+ hogweed) crypto and the gmp math libraries
-find_library(LIBHOGWEED NAMES libhogweed${CMAKE_STATIC_LIBRARY_SUFFIX} hogweed)
+find_library(LIBHOGWEED NAMES libhogweed${CMAKE_STATIC_LIBRARY_SUFFIX} hogweed HINTS /usr/local/lib64)
 find_library(LIBGMP NAMES libgmp${CMAKE_STATIC_LIBRARY_SUFFIX} gmp)
-find_library(LIBNETTLE NAMES libnettle${CMAKE_STATIC_LIBRARY_SUFFIX} nettle)
-find_library(LIBIDN NAMES libidn${CMAKE_STATIC_LIBRARY_SUFFIX} idn)
+find_library(LIBNETTLE NAMES libnettle${CMAKE_STATIC_LIBRARY_SUFFIX} nettle HINTS /usr/local/lib64)
 
-target_link_libraries(pihole-FTL rt Threads::Threads ${LIBHOGWEED} ${LIBGMP} ${LIBNETTLE} ${LIBIDN})
+# for IDN2 we need the idn2 library which in turn depends on the unistring library
+find_library(LIBIDN2 NAMES libidn2${CMAKE_STATIC_LIBRARY_SUFFIX} idn2)
+find_library(LIBUNISTRING NAMES libunistring${CMAKE_STATIC_LIBRARY_SUFFIX} unistring)
+
+target_link_libraries(pihole-FTL rt Threads::Threads ${LIBHOGWEED} ${LIBGMP} ${LIBNETTLE} ${LIBIDN2} ${LIBUNISTRING})
 
 if(LUA_DL STREQUAL "true")
     find_library(LIBDL dl)
@@ -239,17 +338,29 @@ if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
     set(CMAKE_INSTALL_PREFIX "/usr" CACHE PATH "..." FORCE)
 endif()
 
+find_library(LIBMBEDCRYPTO NAMES lmbedcrypto${CMAKE_STATIC_LIBRARY_SUFFIX} mbedcrypto)
+find_library(LIBMBEDX509 NAMES lmbedx509${CMAKE_STATIC_LIBRARY_SUFFIX} mbedx509)
+find_library(LIBMBEDTLS NAMES lmbedtls${CMAKE_STATIC_LIBRARY_SUFFIX} mbedtls)
+if(LIBMBEDCRYPTO AND LIBMBEDX509 AND LIBMBEDTLS)
+    # Link against the mbedTLS libraries, the order is important (!)
+    target_compile_definitions(FTL PRIVATE HAVE_MBEDTLS)
+    target_link_libraries(pihole-FTL ${LIBMBEDTLS} ${LIBMBEDX509} ${LIBMBEDCRYPTO})
+endif()
+
 find_program(SETCAP setcap)
 install(TARGETS pihole-FTL
         RUNTIME DESTINATION bin
         PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
-install(CODE "execute_process(COMMAND ${SETCAP} CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_NICE+eip \$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/bin/pihole-FTL)")
+install(CODE "execute_process(COMMAND ${SETCAP} CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_NICE,CAP_CHOWN+eip \$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/bin/pihole-FTL)")
 
 add_subdirectory(api)
+add_subdirectory(webserver)
+add_subdirectory(zip)
 add_subdirectory(database)
 add_subdirectory(dnsmasq)
 add_subdirectory(lua)
 add_subdirectory(lua/scripts)
 add_subdirectory(tre-regex)
 add_subdirectory(syscalls)
+add_subdirectory(config)
 add_subdirectory(tools)

src/FTL.h

@@ -39,8 +39,6 @@
 #include <syslog.h>
 // tolower()
 #include <ctype.h>
-// Unix socket
-#include <sys/un.h>
 // Interfaces
 #include <ifaddrs.h>
 #include <net/if.h>
@@ -49,6 +47,10 @@
 #define MAX(x,y) (((x) > (y)) ? (x) : (y))
 // MIN(x,y) is already defined in dnsmasq.h
 
+// Number of elements in an array
+#define ArraySize(X) (sizeof(X)/sizeof(X[0]))
+
+// Constant socket buffer length
 #define SOCKETBUFFERLEN 1024
 
 // How often do we garbage collect (to ensure we only have data fitting to the MAXLOGAGE defined above)? [seconds]
@@ -56,7 +58,7 @@
 #define GCinterval 600
 
 // Delay applied to the garbage collecting [seconds]
-// Default: -60 (one minute before a full hour)
+// Default: -60 (one minute before the end of the interval set above)
 #define GCdelay (-60)
 
 // How many client connection do we accept at once?
@@ -102,6 +104,10 @@
 // Default: 1000 (one second)
 #define DATABASE_BUSY_TIMEOUT 1000
 
+// After how much time does a valid API session expire? [seconds]
+// Default: 300 (five minutes)
+#define API_SESSION_EXPIRE 300u
+
 // After how many seconds do we check again if a client can be identified by other means?
 // (e.g., interface, MAC address, hostname)
 // Default: 60 (after one minutee)
@@ -118,12 +124,30 @@
 // Default: 180 [seconds]
 #define DELAY_UPTIME 180
 
+// DB_QUERY_MAX_ITER defines how many queries we check periodically for updates to be added
+// to the in-memory database. This value may need to be increased on *very* busy systems.
+// However, there is an algorithm in place that tries to ensure we are not missing queries
+// on systems with > 100 queries per second
+// Default: 100 (per second)
+#define DB_QUERY_MAX_ITER 100
+
+// Special exit code used to signal that FTL wants to restart
+#define RESTART_FTL_CODE 22
+
+// How often should the database be analyzed?
+// Default: 604800 (once per week)
+#define DATABASE_ANALYZE_INTERVAL 604800
+
+// How often should we update client vendor's from the MAC vendor database?
+// Default: 2592000 (once per month)
+#define DATABASE_MACVENDOR_INTERVAL 2592000
+
 // Use out own syscalls handling functions that will detect possible errors
 // and report accordingly in the log. This will make debugging FTL crash
 // caused by insufficient memory or by code bugs (not properly dealing
 // with NULL pointers) much easier.
 #undef strdup // strdup() is a macro in itself, it needs special handling
-#define free(ptr) FTLfree(ptr, __FILE__,  __FUNCTION__,  __LINE__)
+#define free(ptr) FTLfree((void**)&ptr, __FILE__,  __FUNCTION__,  __LINE__)
 #define strdup(str_in) FTLstrdup(str_in, __FILE__,  __FUNCTION__,  __LINE__)
 #define calloc(numer_of_elements, element_size) FTLcalloc(numer_of_elements, element_size, __FILE__,  __FUNCTION__,  __LINE__)
 #define realloc(ptr, new_size) FTLrealloc(ptr, new_size, __FILE__,  __FUNCTION__,  __LINE__)
@@ -146,10 +170,32 @@
 #define pthread_mutex_lock(mutex) FTLpthread_mutex_lock(mutex, __FILE__,  __FUNCTION__,  __LINE__)
 #define fopen(pathname, mode) FTLfopen(pathname, mode, __FILE__,  __FUNCTION__,  __LINE__)
 #define ftlallocate(fd, offset, len) FTLfallocate(fd, offset, len, __FILE__,  __FUNCTION__,  __LINE__)
+#define strlen(str) FTLstrlen(str, __FILE__,  __FUNCTION__,  __LINE__)
+#define strnlen(str, maxlen) FTLstrnlen(str, maxlen, __FILE__,  __FUNCTION__,  __LINE__)
+#define strcpy(dest, src) FTLstrcpy(dest, src, __FILE__,  __FUNCTION__,  __LINE__)
+#define strncpy(dest, src, n) FTLstrncpy(dest, src, n, __FILE__,  __FUNCTION__,  __LINE__)
+#define memset(s, c, n) FTLmemset(s, c, n, __FILE__,  __FUNCTION__,  __LINE__)
+#define memcpy(dest, src, n) FTLmemcpy(dest, src, n, __FILE__,  __FUNCTION__,  __LINE__)
+#define memmove(dest, src, n) FTLmemmove(dest, src, n, __FILE__,  __FUNCTION__,  __LINE__)
+#define strstr(haystack, needle) FTLstrstr(haystack, needle, __FILE__,  __FUNCTION__,  __LINE__)
+#define strcmp(s1, s2) FTLstrcmp(s1, s2, __FILE__,  __FUNCTION__,  __LINE__)
+#define strncmp(s1, s2, n) FTLstrncmp(s1, s2, n, __FILE__,  __FUNCTION__,  __LINE__)
+#define strcasecmp(s1, s2) FTLstrcasecmp(s1, s2, __FILE__,  __FUNCTION__,  __LINE__)
+#define strncasecmp(s1, s2, n) FTLstrncasecmp(s1, s2, n, __FILE__,  __FUNCTION__,  __LINE__)
+#define strcat(dest, src) FTLstrcat(dest, src, __FILE__,  __FUNCTION__,  __LINE__)
+#define strncat(dest, src, n) FTLstrncat(dest, src, n, __FILE__,  __FUNCTION__,  __LINE__)
+#define memcmp(s1, s2, n) FTLmemcmp(s1, s2, n, __FILE__,  __FUNCTION__,  __LINE__)
+#define memmem(haystack, haystacklen, needle, needlelen) FTLmemmem(haystack, haystacklen, needle, needlelen, __FILE__,  __FUNCTION__,  __LINE__)
 #include "syscalls/syscalls.h"
 
 // Preprocessor help functions
-#define str(x) # x
+#define str(x) #x
 #define xstr(x) str(x)
 
+// Intentionally ignore result of function declared warn_unused_result
+#define igr(x) {__typeof__(x) __attribute__((unused)) d=(x);}
+
+#define max(a,b) ({ __typeof__ (a) _a = (a); __typeof__ (b) _b = (b); _a > _b ? _a : _b; })
+#define min(a,b) ({ __typeof__ (a) _a = (a); __typeof__ (b) _b = (b); _a < _b ? _a : _b; })
+
 #endif // FTL_H

src/api/2fa.c

@@ -0,0 +1,376 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2023 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  API Implementation 2FA methods
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#include "FTL.h"
+#include "api/api.h"
+#include "webserver/json_macros.h"
+#include "log.h"
+#include "config/config.h"
+// getrandom()
+#include "daemon.h"
+// generate_app_password()
+#include "config/password.h"
+
+// TOTP+HMAC
+#include <nettle/hmac.h>
+#include <nettle/sha1.h>
+
+static uint32_t hotp(const uint8_t *key, size_t key_len, const uint64_t counter, const uint8_t digits)
+{
+	// Initialize HMAC-SHA1 (RFC 2104)
+	// TOTP uses HMAC-SHA1 (RFC 6238, section 5.1)
+	struct hmac_sha1_ctx ctx;
+	hmac_sha1_set_key(&ctx, key_len, key);
+
+	// Convert counter to big endian
+	const uint64_t counter_be = htobe64(counter);
+
+	// Compute HMAC-SHA1
+	hmac_sha1_update(&ctx, sizeof(counter_be), (uint8_t*)&counter_be);
+	uint8_t out[SHA1_DIGEST_SIZE];
+	hmac_sha1_digest(&ctx, SHA1_DIGEST_SIZE, out);
+
+	// Truncate HMAC-SHA1 for ease of use
+	// RFC 6238 (section 5.3): offset = last nibble of hash
+	const uint8_t offset = out[SHA1_DIGEST_SIZE-1] & 0x0F;
+	// RFC 6238 (section 5.3): binary = (hash[offset] & 0x7F)   << 24 |
+	//                                  (hash[offset+1] & 0xFF) << 16 |
+	//                                  (hash[offset+2] & 0xFF) << 8 |
+	//                                  (hash[offset+3] & 0xFF)
+	const uint32_t binary = (out[offset] & 0x7F) << 24 |
+	                        (out[offset+1] & 0xFF) << 16 |
+	                        (out[offset+2] & 0xFF) << 8 |
+	                        (out[offset+3] & 0xFF);
+	// RFC 6238 (section 5.3): HOTP = binary mod 10^digits
+	uint32_t mask = 10;
+	for(unsigned int i = 1; i < digits; i++)
+		mask *= 10;
+	return binary % mask;
+}
+
+// RFC 6238 (section 4.1): T0 is the Unix time to start counting time steps
+// (default value is 0, i.e., the Unix epoch) and is also a system parameter.
+#define RFC6238_T0 0
+// RFC 6238 (section 5.2): We RECOMMEND a default time-step size of 30 seconds.
+// This default value of 30 seconds is selected as a balance between security
+// and usability.
+#define RFC6238_X 30
+// RFC 6238 (section 4, R6): The algorithm MUST use a strong shared secret. The
+// length of the shared secret MUST be at least 128 bits (16 Byte). This
+// document RECOMMENDs a shared secret length of 160 bits (20 Byte).
+#define RFC6238_SECRET_LEN 160/8
+// The number of digits to truncate to is not specified in RFC 6238. RFC 4226
+// (section 5.3) specifies that the default is 6 (up to 8) digits, however, the
+// example given in RFC 6238 uses 8 digits.
+#define RFC6238_DIGITS 6
+
+static uint32_t totp(const uint8_t *key, const size_t key_len, const time_t now)
+{
+	// Get time
+	// RFC 6238 (section 4.2): T = (Current Unix time - T0) / X
+	// T is an integer and represents the number of time steps between the
+	// initial time T0 and the current time. T needs to be big endian
+	const uint64_t T = (now - RFC6238_T0) / RFC6238_X;
+
+	// RFC 6238 (section 4.2): TOTP(K, T) = HOTP(K,C) with C = T
+	return hotp(key, key_len, T, RFC6238_DIGITS);
+}
+
+static bool decode_base32_to_uint8_array(const char *base32, uint8_t *out, const size_t out_len)
+{
+	// Base32 alphabet
+	const char *b32 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
+
+	// Check input for validity
+	if(out_len == 0 || out_len*8/5 < strlen(base32) || out_len*8%5 != 0)
+	{
+		log_err("Decoding base32 2FA secret failed, invalid length (%zu)", out_len);
+		return false;
+	}
+
+	// Initialize output array
+	memset(out, 0, out_len);
+
+	// Iterate over input string
+	size_t out_pos = 0u;
+	for(size_t i = 0; i < strlen(base32); i++)
+	{
+		// Get current character
+		const char c = base32[i];
+
+		// Get position of current character in base32 alphabet
+		const char *c_pos = strchr(b32, toupper(c));
+		if(c_pos == NULL)
+		{
+			log_err("Decoding base32 2FA secret failed, invalid character '%c'", c);
+			return false;
+		}
+
+		// Get value of current character
+		const uint8_t c_val = (uint8_t)(c_pos-b32);
+
+		// Iterate over 5 bits of the current character
+		for(unsigned int j = 0; j < 5; j++)
+		{
+			// Current bit position
+			const unsigned int bit = 4-j;
+
+			// Get current bit in the current character
+			const uint8_t c_bit = (c_val >> bit) & 1;
+
+			// Get current byte position in the output array
+			out_pos = (i*5+j)/8;
+
+			// If we are out of bounds, return false
+			if(out_pos >= out_len)
+			{
+				log_err("Decoding base32 2FA secret failed, out of bounds (%zu >= %zu)", out_pos, out_len);
+				return false;
+			}
+
+			// Set current bit in the output array
+			out[out_pos] |= c_bit << (7-((i*5+j)%8));
+		}
+	}
+
+	return true;
+}
+
+static bool encode_uint8_t_array_to_base32(const uint8_t *in, const size_t in_len, char *base32, size_t base32_len)
+{
+	// Base32 alphabet
+	const char *b32 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
+
+	// Check input for validity
+	if(in_len == 0 || in_len > base32_len*5/8 || in_len%5 != 0)
+	{
+		log_err("Encoding base32 2FA secret failed, invalid input length");
+		return false;
+	}
+
+	// Initialize base32 output array
+	memset(base32, 0, base32_len);
+
+	// Iterate over input string
+	size_t base32_pos = 0u;
+	for(size_t i = 0; i < in_len; i++)
+	{
+		// Get current byte
+		const uint8_t b = in[i];
+
+		// Iterate over 8 bits of the current byte
+		for(unsigned int j = 0; j < 8; j++)
+		{
+			// Current bit position
+			const unsigned int bit = 7-j;
+
+			// Get current bit in the current byte
+			const uint8_t b_bit = (b >> bit) & 1;
+
+			// Get current byte position in the base32 output array
+			base32_pos = (i*8+j)/5;
+
+			// If we are out of bounds, return false
+			if(base32_pos >= base32_len)
+			{
+				log_err("Decoding base32 2FA secret failed, base32 output array is too small");
+				return false;
+			}
+
+			// Set current bit in the base32 output array
+			base32[base32_pos] |= b_bit << (4-((i*8+j)%5));
+		}
+	}
+
+	// Iterate over base32 output array and replace each byte with its
+	// corresponding character in the base32 alphabet
+	for(size_t i = 0; i <= base32_pos; i++)
+		base32[i] = b32[(uint8_t)base32[i]];
+
+	return true;
+}
+
+static uint32_t last_code = 0;
+enum totp_status verifyTOTP(const uint32_t incode)
+{
+	// Decode base32 secret
+	uint8_t decoded_secret[RFC6238_SECRET_LEN];
+	if(!decode_base32_to_uint8_array(config.webserver.api.totp_secret.v.s, decoded_secret, sizeof(decoded_secret)))
+		return false;
+
+	// Get current time
+	const time_t now = time(NULL);
+
+	// Verify code for the previous, the current and the next time step
+	for(int i = -1; i <= 1; i++)
+	{
+		const uint32_t gencode = totp(decoded_secret, sizeof(decoded_secret), now + i*RFC6238_X);
+
+		// Verify code
+		// RFC 6238 (section 4.2): If the calculated value matches the value
+		// provided by the user, then the user is authenticated
+		// RFC 6238 (section 4.3): The server MUST NOT accept a TOTP value
+		// generated more than 30 seconds in the future
+		// RFC 6238 (section 4.3): The server MUST NOT accept a TOTP value
+		// generated more than 30 seconds in the past
+		// RFC 6238 (section 4.3): The server MUST NOT accept a TOTP value
+		// it accepted previously
+		if(gencode == incode)
+		{
+			if(gencode == last_code)
+			{
+				log_warn("2FA code has already been used (%i, %u), please wait %lu seconds",
+				         i, gencode, (unsigned long)(RFC6238_X - (now % RFC6238_X)));
+				return TOTP_REUSED;
+			}
+			const char *which = i == -1 ? "previous" : i == 0 ? "current" : "next";
+			log_debug(DEBUG_API, "2FA code from %s time step is valid", which);
+			last_code = gencode;
+			return TOTP_CORRECT;
+		}
+	}
+
+	return TOTP_INVALID;
+}
+
+// Print TOTP code to stdout (for CLI use)
+int printTOTP(void)
+{
+	if(strlen(config.webserver.api.totp_secret.v.s) == 0)
+	{
+		puts("0");
+		return EXIT_SUCCESS;
+	}
+	// Decode base32 secret
+	uint8_t decoded_secret[RFC6238_SECRET_LEN];
+	if(!decode_base32_to_uint8_array(config.webserver.api.totp_secret.v.s, decoded_secret, sizeof(decoded_secret)))
+		return EXIT_FAILURE;
+
+	// Get current time
+	const time_t now = time(NULL);
+	const uint32_t code = totp(decoded_secret, sizeof(decoded_secret), now);
+
+	printf("%u\n", code);
+
+	return EXIT_SUCCESS;
+}
+
+
+// A QR code may be generated from the data using
+// otpauth://totp/<label>?secret=<secret>&issuer=<issuer>&algorithm=<algorithm>&digits=<digits>&period=<period>
+int generateTOTP(struct ftl_conn *api)
+{
+	// Generate random secret using the system's random number generator
+	uint8_t random_secret[RFC6238_SECRET_LEN];
+	if(getrandom(random_secret, sizeof(random_secret), 0) < (ssize_t)sizeof(random_secret))
+	{
+		return send_json_error(api, 500, "internal_error", "Failed to generate random secret", strerror(errno));
+	}
+
+	// Encode base32 secret
+	const size_t base32_len = sizeof(random_secret)*8/5+1;
+	char *base32 = calloc(base32_len, sizeof(char));
+	if(!encode_uint8_t_array_to_base32(random_secret, sizeof(random_secret), base32, base32_len))
+		return send_json_error(api, 500, "internal_error", "Failed to encode secret", "Check FTL.log for details");
+
+	// Create JSON object
+	cJSON *tjson = cJSON_CreateObject();
+	JSON_REF_STR_IN_OBJECT(tjson, "type", "totp");
+	JSON_REF_STR_IN_OBJECT(tjson, "account", config.webserver.domain.v.s);
+	JSON_REF_STR_IN_OBJECT(tjson, "issuer", "Pi-hole%20API");
+	JSON_REF_STR_IN_OBJECT(tjson, "algorithm", "SHA1");
+	JSON_ADD_NUMBER_TO_OBJECT(tjson, "digits", RFC6238_DIGITS);
+	JSON_ADD_NUMBER_TO_OBJECT(tjson, "period", RFC6238_X);
+	JSON_ADD_NUMBER_TO_OBJECT(tjson, "offset", RFC6238_T0);
+	JSON_COPY_STR_TO_OBJECT(tjson, "secret", base32);
+	free(base32);
+	base32 = NULL;
+
+	// Generate a few codes to show the user how to use the secret
+	cJSON *codes = cJSON_CreateArray();
+	for(int i = 0; i < 5; i++)
+	{
+		const time_t now = time(NULL) + (i-1)*RFC6238_X;
+		const uint32_t code = totp(random_secret, sizeof(random_secret), now);
+		JSON_ADD_NUMBER_TO_ARRAY(codes, code);
+	}
+	JSON_ADD_ITEM_TO_OBJECT(tjson, "codes", codes);
+
+	// Send JSON response
+	cJSON *json = cJSON_CreateObject();
+	JSON_ADD_ITEM_TO_OBJECT(json, "totp", tjson);
+	JSON_SEND_OBJECT(json);
+}
+
+int generateAppPw(struct ftl_conn *api)
+{
+	// Generate and set app password
+	char *password = NULL, *pwhash = NULL;
+	if(!generate_app_password(&password, &pwhash))
+	{
+		return send_json_error(api,
+		                       500,
+		                       "internal_error",
+		                       "Failed to generate app password",
+		                       "Check FTL.log for details");
+	}
+
+	// Create JSON object
+	cJSON *tjson = cJSON_CreateObject();
+	JSON_COPY_STR_TO_OBJECT(tjson, "password", password);
+	JSON_COPY_STR_TO_OBJECT(tjson, "hash", pwhash);
+	free(password);
+	password = NULL;
+	free(pwhash);
+	pwhash = NULL;
+
+	// Send JSON response
+	cJSON *json = cJSON_CreateObject();
+	JSON_ADD_ITEM_TO_OBJECT(json, "app", tjson);
+	JSON_SEND_OBJECT(json);
+}
+
+#if 0
+#define RFC6238_TESTKEY "12345678901234567890"
+#define RFC6238_TESTTIME 59
+#define RFC6238_TESTTOTP 94287082
+
+int test_totp(struct ftl_conn *api)
+{
+	// Generate base32 secret
+	uint8_t secret[sizeof(RFC6238_TESTKEY)-1];
+	for(size_t i = 0; i < sizeof(secret); i++)
+		secret[i] = RFC6238_TESTKEY[i];
+
+	// Encode base32 secret
+	char base32_secret[sizeof(secret)*8/5+1];
+	if(!encode_uint8_t_array_to_base32(secret, sizeof(secret), base32_secret, sizeof(base32_secret)))
+		return false;
+
+	// Decode base32 secret
+	uint8_t decoded_secret[sizeof(RFC6238_TESTKEY)-1];
+	if(!decode_base32_to_uint8_array(base32_secret, decoded_secret, sizeof(decoded_secret)))
+		return false;
+
+	// Get test time
+	const time_t now = RFC6238_TESTTIME;
+
+	// Verify code for the current time and the previous and next time step
+	for(int i = -1; i <= 1; i++)
+	{
+		// Verify code
+		const time_t t = now + i*RFC6238_X;
+		if(totp(decoded_secret, sizeof(decoded_secret), t) == RFC6238_TESTTOTP)
+			log_info("Code is valid for time %ld", t);
+	}
+
+	return 200;
+}
+#endif

src/api/CMakeLists.txt

@@ -9,16 +9,32 @@
 # Please see LICENSE file for your rights under this license.
 
 set(sources
-        api.c
+        2fa.c
+        action.c
+        api_helper.h
         api.h
-        msgpack.c
-        request.c
-        request.h
-        socket.c
-        socket.h
+        api.c
+        auth.c
+        auth.h
+        config.c
+        dhcp.c
+        dns.c
+        network.c
+        history.c
+        info.c
+        list.c
+        logs.c
+        queries.c
+        search.c
+        stats_database.c
+        stats.c
+        teleporter.c
+        theme.c
+        theme.h
         )
 
 add_library(api OBJECT ${sources})
 add_dependencies(api gen_version)
 target_compile_options(api PRIVATE ${EXTRAWARN})
 target_include_directories(api PRIVATE ${PROJECT_SOURCE_DIR}/src)
+add_subdirectory(docs)

src/api/action.c

@@ -0,0 +1,177 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2023 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  API Implementation /api/action
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#include "FTL.h"
+#include "webserver/http-common.h"
+#include "webserver/json_macros.h"
+#include "api/api.h"
+// wait()
+#include <sys/wait.h>
+// reboot()
+#include <sys/reboot.h>
+#include <unistd.h>
+// exit_code
+#include "signals.h"
+// flush_network_table()
+#include "database/network-table.h"
+#include "config/config.h"
+
+static int run_and_stream_command(struct ftl_conn *api, const char *path, const char *const args[])
+{
+	// Create a pipe for communication with our child
+	int pipefd[2];
+	if(pipe(pipefd) !=0)
+	{
+		log_err("Cannot create pipe while running gravity action: %s", strerror(errno));
+		return false;
+	}
+
+	// Fork!
+	pid_t cpid = fork();
+	int code = -1;
+	bool crashed = false;
+	if (cpid == 0)
+	{
+		/*** CHILD ***/
+		// Close the reading end of the pipe
+		close(pipefd[0]);
+
+		// Disable logging
+		log_ctrl(false, false);
+
+		// Flush STDERR
+		fflush(stderr);
+
+		// Redirect STDERR into our pipe
+		dup2(pipefd[1], STDERR_FILENO);
+		dup2(pipefd[1], STDOUT_FILENO);
+
+		// Run pihole -g
+		execv(path, (char *const *)args);
+
+		// Exit the fork
+		exit(EXIT_SUCCESS);
+	}
+	else
+	{
+		/*** PARENT ***/
+		// Close the writing end of the pipe
+		close(pipefd[1]);
+
+		// Send 200 OK with chunked size (-1)
+		mg_send_http_ok(api->conn, "text/plain", -1);
+
+		// Read readirected STDOUT/STDERR until EOF
+		// We are only interested in the last pipe line
+		char errbuf[1024] = "";
+		while(read(pipefd[0], errbuf, sizeof(errbuf)) > 0)
+		{
+			// Send chunked data
+			// The chunked size is the length of the string in hex and has to be
+			// transferred in advance, followed by \r\n as line separator and
+			// followed by a chunk of data (the string itself) of the specified
+			// size
+			mg_printf(api->conn, "%zX\r\n%s\r\n", strlen(errbuf), errbuf);
+
+			// Reset buffer
+			memset(errbuf, 0, sizeof(errbuf));
+		}
+
+		// Wait until child has exited to get its return code
+		int status;
+		waitpid(cpid, &status, 0);
+		code = WEXITSTATUS(status);
+
+		if(WIFSIGNALED(status))
+		{
+			crashed = true;
+			log_err("gravity failed with signal %d %s",
+			        WTERMSIG(status),
+			        WCOREDUMP(status) ? "(core dumped)" : "");
+		}
+
+		log_debug(DEBUG_API, "Gravity return code: %d", code);
+
+		// Close the reading end of the pipe
+		close(pipefd[0]);
+	}
+
+	// Send final chunk of size 0 showing end of data
+	mg_printf(api->conn, "0\r\n\r\n");
+
+	if(code == EXIT_SUCCESS && !crashed)
+		return send_json_success(api);
+	else
+		return send_json_error(api, 500,
+		                       "server_error",
+		                       "Gravity failed",
+		                       NULL);
+}
+
+int api_action_gravity(struct ftl_conn *api)
+{
+	return run_and_stream_command(api, "/usr/local/bin/pihole", (const char *const []){ "pihole", "-g", NULL });
+}
+
+int api_action_restartDNS(struct ftl_conn *api)
+{
+	if(!config.webserver.api.allow_destructive.v.b)
+		return send_json_error(api, 403,
+		                       "forbidden",
+		                       "Restarting DNS is not allowed",
+		                       "Check setting webserver.api.allow_destructive");
+
+	log_info("Restarting FTL due to API action request");
+	exit_code = RESTART_FTL_CODE;
+	// Send SIGTERM to FTL
+	kill(main_pid(), SIGTERM);
+
+	return send_json_success(api);
+}
+
+int api_action_flush_logs(struct ftl_conn *api)
+{
+	if(!config.webserver.api.allow_destructive.v.b)
+		return send_json_error(api, 403,
+		                       "forbidden",
+		                       "Flushing the logs is not allowed",
+		                       "Check setting webserver.api.allow_destructive");
+
+	log_info("Received API request to flush the logs");
+
+	// Flush the logs
+	if(flush_dnsmasq_log())
+		return send_json_success(api);
+	else
+		return send_json_error(api, 500,
+		                       "server_error",
+		                       "Cannot flush the logs",
+		                       NULL);
+}
+
+int api_action_flush_arp(struct ftl_conn *api)
+{
+	if(!config.webserver.api.allow_destructive.v.b)
+		return send_json_error(api, 403,
+		                       "forbidden",
+		                       "Flushing the ARP tables is not allowed",
+		                       "Check setting webserver.api.allow_destructive");
+
+	log_info("Received API request to flush the ARP tables");
+
+	// Flush the ARP tables
+	if(flush_network_table())
+		return send_json_success(api);
+	else
+		return send_json_error(api, 500,
+		                       "server_error",
+		                       "Cannot flush the ARP tables",
+		                       NULL);
+}

src/api/api.h

@@ -3,51 +3,126 @@
 *  Network-wide ad blocking via your own hardware.
 *
 *  FTL Engine
-*  API commands and MessagePack helpers
+*  API route prototypes
 *
 *  This file is copyright under the latest version of the EUPL.
 *  Please see LICENSE file for your rights under this license. */
-#ifndef API_H
-#define API_H
+#ifndef ROUTES_H
+#define ROUTES_H
+
+// struct mg_connection
+#include "webserver/civetweb/civetweb.h"
+// type cJSON
+#include "webserver/cJSON/cJSON.h"
+#include "webserver/http-common.h"
+// regex_t
+#include "regex_r.h"
+
+// Common definitions
+#define LOCALHOSTv4 "127.0.0.1"
+#define LOCALHOSTv6 "::1"
+
+// API router
+int api_handler(struct mg_connection *conn, void *ignored);
 
 // Statistic methods
-void getStats(const int sock, const bool istelnet);
-void getOverTime(const int sock, const bool istelnet);
-void getTopDomains(const char *client_message, const int sock, const bool istelnet);
-void getTopClients(const char *client_message, const int sock, const bool istelnet);
-void getUpstreamDestinations(const char *client_message, const int sock, const bool istelnet);
-void getQueryTypes(const int sock, const bool istelnet);
-void getAllQueries(const char *client_message, const int sock, const bool istelnet);
-void getRecentBlocked(const char *client_message, const int sock, const bool istelnet);
-void getClientsOverTime(const int sock, const bool istelnet);
-void getClientNames(const int sock, const bool istelnet);
+int __attribute__((pure)) cmpdesc(const void *a, const void *b);
+int api_stats_summary(struct ftl_conn *api);
+int api_stats_query_types(struct ftl_conn *api);
+int api_stats_upstreams(struct ftl_conn *api);
+int api_stats_top_domains(struct ftl_conn *api);
+int api_stats_top_clients(struct ftl_conn *api);
+int api_stats_recentblocked(struct ftl_conn *api);
 
-// FTL methods
-void getClientID(const int sock, const bool istelnet);
-void getVersion(const int sock, const bool istelnet);
-void getDBstats(const int sock, const bool istelnet);
-void getUnknownQueries(const int sock, const bool istelnet);
-void getMAXLOGAGE(const int sock);
-void getGateway(const int sock);
-void getInterfaces(const int sock);
+// History methods
+int api_history(struct ftl_conn *api);
+int api_history_clients(struct ftl_conn *api);
 
-// DNS resolver methods (dnsmasq_interface.c)
-void getCacheInformation(const int sock);
-void getDNSport(const int sock);
+// History methods (database)
+int api_history_database(struct ftl_conn *api);
+int api_history_database_clients(struct ftl_conn *api);
 
-// MessagePack serialization helpers
-void pack_eom(const int sock);
-void pack_bool(const int sock, const bool value);
-void pack_uint8(const int sock, const uint8_t value);
-void pack_uint64(const int sock, const uint64_t value);
-void pack_int32(const int sock, const int32_t value);
-void pack_int64(const int sock, const int64_t value);
-void pack_float(const int sock, const float value);
-bool pack_fixstr(const int sock, const char *string);
-bool pack_str32(const int sock, const char *string);
-void pack_map16_start(const int sock, const uint16_t length);
+// Query methods
+int api_queries(struct ftl_conn *api);
+int api_queries_suggestions(struct ftl_conn *api);
+bool compile_filter_regex(struct ftl_conn *api, const char *path, cJSON *json, regex_t **regex, unsigned int *N_regex);
 
-// DHCP lease management
-void delete_lease(const char *client_message, const int sock);
+// Statistics methods (database)
+int api_stats_database_top_items(struct ftl_conn *api);
+int api_stats_database_summary(struct ftl_conn *api);
+int api_stats_database_query_types(struct ftl_conn *api);
+int api_stats_database_upstreams(struct ftl_conn *api);
 
-#endif // API_H
+// Info methods
+int api_info_client(struct ftl_conn *api);
+int api_info_database(struct ftl_conn *api);
+int api_info_system(struct ftl_conn *api);
+int api_info_ftl(struct ftl_conn *api);
+int api_info_host(struct ftl_conn *api);
+int api_info_sensors(struct ftl_conn *api);
+int api_info_version(struct ftl_conn *api);
+int api_info_messages_count(struct ftl_conn *api);
+int api_info_messages(struct ftl_conn *api);
+int api_info_metrics(struct ftl_conn *api);
+int api_info_login(struct ftl_conn *api);
+
+// Config methods
+int api_config(struct ftl_conn *api);
+
+// Log methods
+int api_logs(struct ftl_conn *api);
+
+// Network methods
+int api_network_gateway(struct ftl_conn *api);
+int api_network_interfaces(struct ftl_conn *api);
+int api_network_devices(struct ftl_conn *api);
+int api_client_suggestions(struct ftl_conn *api);
+
+// DNS methods
+int api_dns_blocking(struct ftl_conn *api);
+
+// List methods
+int api_list(struct ftl_conn *api);
+int api_group(struct ftl_conn *api);
+
+// Auth method
+void init_api(void);
+void free_api(void);
+int check_client_auth(struct ftl_conn *api, const bool is_api);
+int api_auth(struct ftl_conn *api);
+void delete_all_sessions(void);
+int api_auth_sessions(struct ftl_conn *api);
+int api_auth_session_delete(struct ftl_conn *api);
+bool is_local_api_user(const char *remote_addr) __attribute__((pure));
+
+// 2FA methods
+enum totp_status {
+	TOTP_INVALID,
+	TOTP_CORRECT,
+	TOTP_REUSED,
+} __attribute__ ((packed));
+enum totp_status verifyTOTP(const uint32_t code);
+int generateTOTP(struct ftl_conn *api);
+int printTOTP(void);
+int generateAppPw(struct ftl_conn *api);
+
+// Documentation methods
+int api_docs(struct ftl_conn *api);
+
+// Teleporter methods
+int api_teleporter(struct ftl_conn *api);
+
+// Action methods
+int api_action_gravity(struct ftl_conn *api);
+int api_action_restartDNS(struct ftl_conn *api);
+int api_action_flush_logs(struct ftl_conn *api);
+int api_action_flush_arp(struct ftl_conn *api);
+
+// Search methods
+int api_search(struct ftl_conn *api);
+
+// DHCP methods
+int api_dhcp_leases_GET(struct ftl_conn *api);
+int api_dhcp_leases_DELETE(struct ftl_conn *api);
+
+#endif // ROUTES_H

src/api/auth.c

@@ -0,0 +1,706 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2019 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  API Implementation /api/auth
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#include "FTL.h"
+#include "api/auth.h"
+#include "webserver/http-common.h"
+#include "webserver/json_macros.h"
+#include "api/api.h"
+#include "log.h"
+#include "config/config.h"
+// get_password_hash()
+#include "config/setupVars.h"
+// (un)lock_shm()
+#include "shmem.h"
+// getrandom()
+#include "daemon.h"
+// sha256_raw_to_hex()
+#include "config/password.h"
+// database session functions
+#include "database/session-table.h"
+
+static uint16_t max_sessions = 0;
+static struct session *auth_data = NULL;
+
+static void add_request_info(struct ftl_conn *api, const char *csrf)
+{
+	// Copy CSRF token into request
+	if(csrf != NULL)
+		strncpy((char*)api->request->csrf_token, csrf, sizeof(api->request->csrf_token) - 1);
+
+	// Store that this client is authenticated
+	// We use memset() with the size of an int here to avoid a
+	// compiler warning about modifying a variable in a const struct
+	memset((int*)&api->request->is_authenticated, 1, sizeof(api->request->is_authenticated));
+}
+
+void init_api(void)
+{
+	// Restore sessions from database
+	max_sessions = config.webserver.api.max_sessions.v.u16;
+	auth_data = calloc(max_sessions, sizeof(struct session));
+	if(auth_data == NULL)
+	{
+		log_crit("Could not allocate memory for API sessions, check config value of webserver.api.max_sessions");
+		exit(EXIT_FAILURE);
+	}
+	restore_db_sessions(auth_data, max_sessions);
+}
+
+void free_api(void)
+{
+	if(auth_data == NULL)
+		return;
+
+	// Store sessions in database
+	backup_db_sessions(auth_data, max_sessions);
+	max_sessions = 0;
+	free(auth_data);
+	auth_data = NULL;
+}
+
+// Is this client connecting from localhost?
+bool __attribute__((pure)) is_local_api_user(const char *remote_addr)
+{
+	return strcmp(remote_addr, LOCALHOSTv4) == 0 ||
+	       strcmp(remote_addr, LOCALHOSTv6) == 0;
+}
+
+// Can we validate this client?
+// Returns -1 if not authenticated or expired
+// Returns >= 0 for any valid authentication
+int check_client_auth(struct ftl_conn *api, const bool is_api)
+{
+	// Is the user requesting from localhost?
+	// This may be allowed without authentication depending on the configuration
+	if(!config.webserver.api.localAPIauth.v.b && is_local_api_user(api->request->remote_addr))
+	{
+		api->message = "no auth required for local user";
+		add_request_info(api, NULL);
+		return API_AUTH_LOCALHOST;
+	}
+
+	// When the pwhash is unset, authentication is disabled
+	if(config.webserver.api.pwhash.v.s[0] == '\0')
+	{
+		api->message = "no password set";
+		add_request_info(api, NULL);
+		return API_AUTH_EMPTYPASS;
+	}
+
+	// Does the client provide a session ID?
+	char sid[SID_SIZE];
+	const char *sid_source = "-";
+	// Try to extract SID from cookie
+	bool sid_avail = false;
+
+	// If not, does the client provide a session ID via GET/POST?
+	if(api->payload.avail)
+	{
+		// Try to extract SID from form-encoded payload
+		if(GET_VAR("sid", sid, api->payload.raw) > 0)
+		{
+			// "+" may have been replaced by " ", undo this here
+			for(unsigned int i = 0; i < SID_SIZE; i++)
+				if(sid[i] == ' ')
+					sid[i] = '+';
+
+			// Zero terminate SID string
+			sid[SID_SIZE-1] = '\0';
+			// Mention source of SID
+			sid_source = "payload (form-data)";
+			// Mark SID as available
+			sid_avail = true;
+		}
+		// Try to extract SID from root of a possibly included JSON payload
+		else if(api->payload.json != NULL)
+		{
+			cJSON *sid_obj = cJSON_GetObjectItem(api->payload.json, "sid");
+			if(cJSON_IsString(sid_obj))
+			{
+				// Copy SID string
+				strncpy(sid, sid_obj->valuestring, SID_SIZE - 1u);
+				// Zero terminate SID string
+				sid[SID_SIZE-1] = '\0';
+				// Mention source of SID
+				sid_source = "payload (JSON)";
+				// Mark SID as available
+				sid_avail = true;
+			}
+		}
+	}
+
+	// If not, does the client provide a session ID via HEADER?
+	if(!sid_avail)
+	{
+		const char *sid_header = NULL;
+		// Try to extract SID from header
+		if((sid_header = mg_get_header(api->conn, "sid")) != NULL ||
+		   (sid_header = mg_get_header(api->conn, "X-FTL-SID")) != NULL)
+		{
+			// Copy SID string
+			strncpy(sid, sid_header, SID_SIZE - 1u);
+			// Zero terminate SID string
+			sid[SID_SIZE-1] = '\0';
+			// Mention source of SID
+			sid_source = "header";
+			// Mark SID as available
+			sid_avail = true;
+		}
+	}
+
+	// If not, does the client provide a session ID via COOKIE?
+	bool cookie_auth = false;
+	if(!sid_avail)
+	{
+		cookie_auth = http_get_cookie_str(api, "sid", sid, SID_SIZE);
+		if(cookie_auth)
+		{
+			// Mention source of SID
+			sid_source = "cookie";
+			// Mark SID as available
+			sid_avail = true;
+		}
+	}
+
+	// If not, does the client provide a session ID via URI?
+	if(!sid_avail && api->request->query_string && GET_VAR("sid", sid, api->request->query_string) > 0)
+	{
+		// "+" may have been replaced by " ", undo this here
+		for(unsigned int i = 0; i < SID_SIZE; i++)
+			if(sid[i] == ' ')
+				sid[i] = '+';
+
+		// Zero terminate SID string
+		sid[SID_SIZE-1] = '\0';
+		// Mention source of SID
+		sid_source = "URI";
+		// Mark SID as available
+		sid_avail = true;
+	}
+
+	if(!sid_avail)
+	{
+		api->message = "no SID provided";
+		log_debug(DEBUG_API, "API Authentication: FAIL (%s)", api->message);
+		return API_AUTH_UNAUTHORIZED;
+	}
+
+	// else: Analyze SID
+	int user_id = API_AUTH_UNAUTHORIZED;
+	const time_t now = time(NULL);
+	log_debug(DEBUG_API, "Read sid=\"%s\" from %s", sid, sid_source);
+
+	// If the SID has been sent through a cookie, we require a CSRF token in
+	// the header to be sent along with the request for any API requests
+	char csrf[SID_SIZE];
+	const bool need_csrf = cookie_auth && is_api;
+	if(need_csrf)
+	{
+		const char *csrf_header = NULL;
+		// Try to extract CSRF token from header
+		if((csrf_header = mg_get_header(api->conn, "X-CSRF-TOKEN")) != NULL)
+		{
+			// Copy CSRF string
+			strncpy(csrf, csrf_header, SID_SIZE - 1u);
+			// Zero terminate CSRF string
+			csrf[SID_SIZE-1] = '\0';
+		}
+		else
+		{
+			api->message = "Cookie authentication without CSRF token";
+			log_debug(DEBUG_API, "API Authentication: FAIL (%s)", api->message);
+			return API_AUTH_UNAUTHORIZED;
+		}
+	}
+
+	bool expired = false;
+	for(unsigned int i = 0; i < max_sessions; i++)
+	{
+		if(auth_data[i].used &&
+		   strcmp(auth_data[i].sid, sid) == 0)
+		{
+			// Check if session is known but expired
+			if(auth_data[i].valid_until < now)
+				expired = true;
+
+			// Check CSRF if authentiating via cookie
+			if(need_csrf && strcmp(auth_data[i].csrf, csrf) != 0)
+			{
+				api->message = "CSRF token mismatch";
+				log_debug(DEBUG_API, "API Authentication: FAIL (%s, received \"%s\", expected \"%s\")",
+				          api->message, csrf, auth_data[i].csrf);
+				return API_AUTH_UNAUTHORIZED;
+			}
+			user_id = i;
+			break;
+		}
+	}
+	if(user_id > API_AUTH_UNAUTHORIZED)
+	{
+		// Authentication successful: valid session
+
+		// Update timestamp of this client to extend
+		// the validity of their API authentication
+		auth_data[user_id].valid_until = now + config.webserver.session.timeout.v.ui;
+
+		// Set strict_tls permanently to false if the client connected via HTTP
+		auth_data[user_id].tls.mixed |= api->request->is_ssl != auth_data[user_id].tls.login;
+
+		// Update user cookie
+		if(snprintf(pi_hole_extra_headers, sizeof(pi_hole_extra_headers),
+		            FTL_SET_COOKIE,
+		            auth_data[user_id].sid, config.webserver.session.timeout.v.ui) < 0)
+		{
+			return send_json_error(api, 500, "internal_error", "Internal server error", NULL);
+		}
+
+		// Add CSRF token to request
+		add_request_info(api, auth_data[user_id].csrf);
+
+		// Debug logging
+		if(config.debug.api.v.b)
+		{
+			char timestr[128];
+			get_timestr(timestr, auth_data[user_id].valid_until, false, false);
+			log_debug(DEBUG_API, "Recognized known user: user_id %i, valid_until: %s, remote_addr %s (%s at login)",
+			          user_id, timestr, api->request->remote_addr, auth_data[user_id].remote_addr);
+		}
+	}
+	else
+	{
+		api->message = expired ? "session expired" : "session unknown";
+		log_debug(DEBUG_API, "API Authentication: FAIL (%s)", api->message);
+		return API_AUTH_UNAUTHORIZED;
+	}
+
+	api->user_id = user_id;
+
+	api->message = "correct password";
+	return user_id;
+}
+
+static int get_all_sessions(struct ftl_conn *api, cJSON *json)
+{
+	const time_t now = time(NULL);
+	cJSON *sessions = JSON_NEW_ARRAY();
+	for(int i = 0; i < max_sessions; i++)
+	{
+		if(!auth_data[i].used)
+			continue;
+		cJSON *session = JSON_NEW_OBJECT();
+		JSON_ADD_NUMBER_TO_OBJECT(session, "id", i);
+		JSON_ADD_BOOL_TO_OBJECT(session, "current_session", i == api->user_id);
+		JSON_ADD_BOOL_TO_OBJECT(session, "valid", auth_data[i].valid_until >= now);
+		cJSON *tls = JSON_NEW_OBJECT();
+		JSON_ADD_BOOL_TO_OBJECT(tls, "login", auth_data[i].tls.login);
+		JSON_ADD_BOOL_TO_OBJECT(tls, "mixed", auth_data[i].tls.mixed);
+		JSON_ADD_ITEM_TO_OBJECT(session, "tls", tls);
+		JSON_ADD_NUMBER_TO_OBJECT(session, "login_at", auth_data[i].login_at);
+		JSON_ADD_NUMBER_TO_OBJECT(session, "last_active", auth_data[i].valid_until - config.webserver.session.timeout.v.ui);
+		JSON_ADD_NUMBER_TO_OBJECT(session, "valid_until", auth_data[i].valid_until);
+		JSON_REF_STR_IN_OBJECT(session, "remote_addr", auth_data[i].remote_addr);
+		JSON_REF_STR_IN_OBJECT(session, "user_agent", auth_data[i].user_agent);
+		JSON_ADD_BOOL_TO_OBJECT(session, "app", auth_data[i].app);
+		JSON_ADD_ITEM_TO_ARRAY(sessions, session);
+	}
+	JSON_ADD_ITEM_TO_OBJECT(json, "sessions", sessions);
+	return 0;
+}
+
+static int get_session_object(struct ftl_conn *api, cJSON *json, const int user_id, const time_t now)
+{
+	cJSON *session = JSON_NEW_OBJECT();
+
+	// Authentication not needed
+	if(user_id == API_AUTH_LOCALHOST || user_id == API_AUTH_EMPTYPASS)
+	{
+		JSON_ADD_BOOL_TO_OBJECT(session, "valid", true);
+		JSON_ADD_BOOL_TO_OBJECT(session, "totp", strlen(config.webserver.api.totp_secret.v.s) > 0);
+		JSON_ADD_NULL_TO_OBJECT(session, "sid");
+		JSON_ADD_NUMBER_TO_OBJECT(session, "validity", -1);
+		JSON_REF_STR_IN_OBJECT(session, "message", api->message);
+		JSON_ADD_ITEM_TO_OBJECT(json, "session", session);
+		return 0;
+	}
+
+	// Valid session
+	if(user_id > API_AUTH_UNAUTHORIZED && auth_data[user_id].used)
+	{
+		JSON_ADD_BOOL_TO_OBJECT(session, "valid", true);
+		JSON_ADD_BOOL_TO_OBJECT(session, "totp", strlen(config.webserver.api.totp_secret.v.s) > 0);
+		JSON_REF_STR_IN_OBJECT(session, "sid", auth_data[user_id].sid);
+		JSON_REF_STR_IN_OBJECT(session, "csrf", auth_data[user_id].csrf);
+		JSON_ADD_NUMBER_TO_OBJECT(session, "validity", auth_data[user_id].valid_until - now);
+		JSON_REF_STR_IN_OBJECT(session, "message", api->message);
+		JSON_ADD_ITEM_TO_OBJECT(json, "session", session);
+		return 0;
+	}
+
+	// No valid session
+	JSON_ADD_BOOL_TO_OBJECT(session, "valid", false);
+	JSON_ADD_BOOL_TO_OBJECT(session, "totp", strlen(config.webserver.api.totp_secret.v.s) > 0);
+	JSON_ADD_NULL_TO_OBJECT(session, "sid");
+	JSON_ADD_NUMBER_TO_OBJECT(session, "validity", -1);
+	JSON_REF_STR_IN_OBJECT(session, "message", api->message);
+	JSON_ADD_ITEM_TO_OBJECT(json, "session", session);
+	return 0;
+}
+
+static bool delete_session(const int user_id)
+{
+	// Skip if nothing to be done here
+	if(user_id < 0 || user_id >= max_sessions)
+		return false;
+
+	const bool was_valid = auth_data[user_id].used;
+
+	// Zero out this session (also sets valid to false == 0)
+	memset(&auth_data[user_id], 0, sizeof(auth_data[user_id]));
+
+	return was_valid;
+}
+
+void delete_all_sessions(void)
+{
+	// Zero out all sessions without looping
+	memset(auth_data, 0, max_sessions*sizeof(*auth_data));
+}
+
+static int send_api_auth_status(struct ftl_conn *api, const int user_id, const time_t now)
+{
+	if(user_id > API_AUTH_UNAUTHORIZED && (api->method == HTTP_GET || api->method == HTTP_POST))
+	{
+		log_debug(DEBUG_API, "API Auth status: OK");
+
+		// Ten minutes validity
+		if(snprintf(pi_hole_extra_headers, sizeof(pi_hole_extra_headers),
+		            FTL_SET_COOKIE,
+		            auth_data[user_id].sid, config.webserver.session.timeout.d.ui) < 0)
+		{
+			return send_json_error(api, 500, "internal_error", "Internal server error", NULL);
+		}
+
+		cJSON *json = JSON_NEW_OBJECT();
+		get_session_object(api, json, user_id, now);
+		JSON_SEND_OBJECT(json);
+	}
+	else if(api->method == HTTP_DELETE)
+	{
+		if(user_id > API_AUTH_UNAUTHORIZED)
+		{
+			log_debug(DEBUG_API, "API Auth status: Logout, asking to delete cookie");
+
+			strncpy(pi_hole_extra_headers, FTL_DELETE_COOKIE, sizeof(pi_hole_extra_headers));
+
+			// Revoke client authentication. This slot can be used by a new client afterwards.
+			const int code = delete_session(user_id) ? 204 : 404;
+
+			// Send empty reply with appropriate HTTP status code
+			send_http_code(api, "application/json; charset=utf-8", code, "");
+			return code;
+		}
+		else
+		{
+			log_debug(DEBUG_API, "API Auth status: Logout, but not authenticated");
+
+			cJSON *json = JSON_NEW_OBJECT();
+			get_session_object(api, json, user_id, now);
+			JSON_SEND_OBJECT_CODE(json, 401); // 401 Unauthorized
+		}
+	}
+	else if(user_id == API_AUTH_LOCALHOST)
+	{
+		log_debug(DEBUG_API, "API Auth status: OK (localhost does not need auth)");
+
+		cJSON *json = JSON_NEW_OBJECT();
+		get_session_object(api, json, user_id, now);
+		JSON_SEND_OBJECT(json);
+	}
+	else if(user_id == API_AUTH_EMPTYPASS)
+	{
+		log_debug(DEBUG_API, "API Auth status: OK (empty password)");
+
+		cJSON *json = JSON_NEW_OBJECT();
+		get_session_object(api, json, user_id, now);
+		JSON_SEND_OBJECT(json);
+	}
+	else
+	{
+		log_debug(DEBUG_API, "API Auth status: Invalid, asking to delete cookie");
+
+		strncpy(pi_hole_extra_headers, FTL_DELETE_COOKIE, sizeof(pi_hole_extra_headers));
+		cJSON *json = JSON_NEW_OBJECT();
+		get_session_object(api, json, user_id, now);
+		JSON_SEND_OBJECT_CODE(json, 401); // 401 Unauthorized
+	}
+}
+
+static void generateSID(char *sid)
+{
+	uint8_t raw_sid[SID_SIZE];
+	if(getrandom(raw_sid, sizeof(raw_sid), 0) < 0)
+	{
+		log_err("getrandom() failed in generateSID()");
+		return;
+	}
+	base64_encode_raw(NETTLE_SIGN sid, SID_BITSIZE/8, raw_sid);
+	sid[SID_SIZE-1] = '\0';
+}
+
+// api/auth
+//  GET: Check authentication
+//  POST: Login
+//  DELETE: Logout
+int api_auth(struct ftl_conn *api)
+{
+	// Check HTTP method
+	char *password = NULL;
+	const time_t now = time(NULL);
+	const bool empty_password = config.webserver.api.pwhash.v.s[0] == '\0';
+
+	if(api->item != NULL && strlen(api->item) > 0)
+	{
+		// Sub-paths are not allowed
+		return 0;
+	}
+
+	// Login attempt, check password
+	if(api->method == HTTP_POST)
+	{
+		// Try to extract response from payload
+		const int ret = check_json_payload(api);
+		if(ret != 0)
+			return ret;
+
+		// Check if password is available
+		cJSON *json_password;
+		if((json_password = cJSON_GetObjectItemCaseSensitive(api->payload.json, "password")) == NULL)
+		{
+			const char *message = "No password found in JSON payload";
+			log_debug(DEBUG_API, "API auth error: %s", message);
+			return send_json_error(api, 400,
+			                       "bad_request",
+			                       message,
+			                       NULL);
+		}
+
+		// Check password type
+		if(!cJSON_IsString(json_password))
+		{
+			const char *message = "Field password has to be of type 'string'";
+			log_debug(DEBUG_API, "API auth error: %s", message);
+			return send_json_error(api, 400,
+			                       "bad_request",
+			                       message,
+			                       NULL);
+		}
+
+		// password is already null-terminated
+		password = json_password->valuestring;
+	}
+
+	// Did the client authenticate before and we can validate this?
+	int user_id = check_client_auth(api, false);
+
+	// If this is a valid session, we can exit early at this point if no password is supplied
+	if(user_id != API_AUTH_UNAUTHORIZED && (password == NULL || strlen(password) == 0))
+		return send_api_auth_status(api, user_id, now);
+
+	// Logout attempt
+	if(api->method == HTTP_DELETE)
+	{
+		log_debug(DEBUG_API, "API Auth: User with ID %i wants to log out", user_id);
+		return send_api_auth_status(api, user_id, now);
+	}
+
+	// If this is not a login attempt, we can exit early at this point
+	if(password == NULL && !empty_password)
+		return send_api_auth_status(api, user_id, now);
+
+	// else: Login attempt
+	// - Client tries to authenticate using a password, or
+	// - There no password on this machine
+	enum password_result result = PASSWORD_INCORRECT;
+
+	// If there is no password (or empty), check if there is any password at all
+	if(empty_password && (password == NULL || strlen(password) == 0))
+		result = PASSWORD_CORRECT;
+	else
+		result = verify_login(password);
+
+	if(result == PASSWORD_CORRECT || result == APPPASSWORD_CORRECT)
+	{
+		// Accepted
+
+		// Zero-out password in memory to avoid leaking it when it is
+		// freed at the end of the current API request
+		if(password != NULL)
+			memset(password, 0, strlen(password));
+
+		// Check possible 2FA token
+		// Successful login with empty password does not require 2FA
+		if(strlen(config.webserver.api.totp_secret.v.s) > 0 && result != APPPASSWORD_CORRECT)
+		{
+			// Get 2FA token from payload
+			cJSON *json_totp;
+			if((json_totp = cJSON_GetObjectItemCaseSensitive(api->payload.json, "totp")) == NULL)
+			{
+				const char *message = "No 2FA token found in JSON payload";
+				log_debug(DEBUG_API, "API auth error: %s", message);
+				return send_json_error(api, 400,
+							"bad_request",
+							message,
+							NULL);
+			}
+
+			enum totp_status totp = verifyTOTP(json_totp->valueint);
+			if(totp == TOTP_REUSED)
+			{
+				// 2FA token has been reused
+				return send_json_error(api, 401,
+				                       "unauthorized",
+				                       "Reused 2FA token",
+				                       "wait for new token");
+			}
+			else if(totp != TOTP_CORRECT)
+			{
+				// 2FA token is invalid
+				return send_json_error(api, 401,
+				                       "unauthorized",
+				                       "Invalid 2FA token",
+				                       NULL);
+			}
+		}
+
+		// Find unused authentication slot
+		for(unsigned int i = 0; i < max_sessions; i++)
+		{
+			// Expired slow, mark as unused
+			if(auth_data[i].used &&
+			   auth_data[i].valid_until < now)
+			{
+				log_debug(DEBUG_API, "API: Session of client %u (%s) expired, freeing...",
+						i, auth_data[i].remote_addr);
+				delete_session(i);
+			}
+
+			// Found unused authentication slot (might have been freed before)
+			if(!auth_data[i].used)
+			{
+				// Mark as used
+				auth_data[i].used = true;
+				// Set validitiy to now + timeout
+				auth_data[i].login_at = now;
+				auth_data[i].valid_until = now + config.webserver.session.timeout.v.ui;
+				// Set remote address
+				strncpy(auth_data[i].remote_addr, api->request->remote_addr, sizeof(auth_data[i].remote_addr));
+				auth_data[i].remote_addr[sizeof(auth_data[i].remote_addr)-1] = '\0';
+				// Store user-agent (if available)
+				const char *user_agent = mg_get_header(api->conn, "user-agent");
+				if(user_agent != NULL)
+				{
+					strncpy(auth_data[i].user_agent, user_agent, sizeof(auth_data[i].user_agent));
+					auth_data[i].user_agent[sizeof(auth_data[i].user_agent)-1] = '\0';
+				}
+				else
+				{
+					auth_data[i].user_agent[0] = '\0';
+				}
+
+				auth_data[i].tls.login = api->request->is_ssl;
+				auth_data[i].tls.mixed = false;
+				auth_data[i].app = result == APPPASSWORD_CORRECT;
+
+				// Generate new SID and CSRF token
+				generateSID(auth_data[i].sid);
+				generateSID(auth_data[i].csrf);
+
+				user_id = i;
+				break;
+			}
+		}
+
+		// Debug logging
+		if(config.debug.api.v.b && user_id > API_AUTH_UNAUTHORIZED)
+		{
+			char timestr[128];
+			get_timestr(timestr, auth_data[user_id].valid_until, false, false);
+			log_debug(DEBUG_API, "API: Registered new user: user_id %i valid_until: %s remote_addr %s (accepted due to %s)",
+					user_id, timestr, auth_data[user_id].remote_addr,
+					empty_password ? "empty password" : "correct response");
+		}
+		if(user_id == API_AUTH_UNAUTHORIZED)
+		{
+			log_warn("No free API seats available (webserver.api.max_sessions = %u), not authenticating client",
+			         max_sessions);
+
+			return send_json_error(api, 429,
+			                       "api_seats_exceeded",
+			                       "API seats exceeded",
+			                       "increase webserver.api.max_sessions");
+		}
+
+		api->message = result == APPPASSWORD_CORRECT ? "app-password correct" : "password correct";
+	}
+	else if(result == PASSWORD_RATE_LIMITED)
+	{
+		// Rate limited
+		return send_json_error(api, 429,
+		                       "rate_limiting",
+		                       "Rate-limiting login attempts",
+		                       NULL);
+	}
+	else if(result == NO_PASSWORD_SET)
+	{
+		// No password set
+		api->message = "password incorrect";
+		log_debug(DEBUG_API, "API: Trying to auth with password but none set: '%s'", password);
+	}
+	else
+	{
+		api->message = "password incorrect";
+		log_debug(DEBUG_API, "API: Password incorrect: '%s'", password);
+	}
+
+	// Free allocated memory
+	return send_api_auth_status(api, user_id, now);
+}
+
+int api_auth_sessions(struct ftl_conn *api)
+{
+	// Get session object
+	cJSON *json = JSON_NEW_OBJECT();
+	get_all_sessions(api, json);
+	JSON_SEND_OBJECT(json);
+}
+
+int api_auth_session_delete(struct ftl_conn *api)
+{
+	// Get user ID
+	int uid;
+	if(sscanf(api->item, "%i", &uid) != 1)
+		return send_json_error(api, 400, "bad_request", "Missing or invalid session ID", NULL);
+
+	// Check if session ID is valid
+	if(uid <= API_AUTH_UNAUTHORIZED || uid >= max_sessions)
+		return send_json_error(api, 400, "bad_request", "Session ID out of bounds", NULL);
+
+	// Check if session is used
+	if(!auth_data[uid].used)
+		return send_json_error(api, 400, "bad_request", "Session ID not in use", NULL);
+
+	// Delete session
+	const int code = delete_session(uid) ? 204 : 404;
+
+	// Send empty reply with appropriate HTTP status code
+	send_http_code(api, "application/json; charset=utf-8", code, "");
+	return code;
+}

src/api/auth.h

@@ -0,0 +1,63 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2023 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  API authentication prototypes
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#ifndef AUTH_H
+#define AUTH_H
+
+// crypto library
+#include <nettle/sha2.h>
+#include <nettle/base64.h>
+#include <nettle/version.h>
+
+// On 2017-08-27 (after v3.3, before v3.4), nettle changed the type of
+// destination from uint_8t* to char* in all base64 and base16 functions
+// (armor-signedness branch). This is a breaking change as this is a change in
+// signedness causing issues when compiling FTL against older versions of
+// nettle. We create this constant here to have a conversion if necessary.
+// See https://github.com/gnutls/nettle/commit/f2da403135e2b2f641cf0f8219ad5b72083b7dfd
+#if NETTLE_VERSION_MAJOR == 3 && NETTLE_VERSION_MINOR < 4
+#define NETTLE_SIGN (uint8_t*)
+#else
+#define NETTLE_SIGN
+#endif
+
+// How many bits should the SID and CSRF token use?
+#define SID_BITSIZE 128
+#define SID_SIZE BASE64_ENCODE_RAW_LENGTH(SID_BITSIZE/8)
+
+// SameSite=Strict: Defense against some classes of cross-site request forgery
+// (CSRF) attacks. This ensures the session cookie will only be sent in a
+// first-party (i.e., Pi-hole) context and NOT be sent along with requests
+// initiated by third party websites.
+//
+// HttpOnly: the cookie cannot be accessed through client side script (if the
+// browser supports this flag). As a result, even if a cross-site scripting
+// (XSS) flaw exists, and a user accidentally accesses a link that exploits this
+// flaw, the browser (primarily Internet Explorer) will not reveal the cookie to
+// a third party.
+#define FTL_SET_COOKIE "Set-Cookie: sid=%s; SameSite=Strict; Path=/; Max-Age=%u; HttpOnly\r\n"
+#define FTL_DELETE_COOKIE "Set-Cookie: sid=deleted; SameSite=Strict; Path=/; Max-Age=-1\r\n"
+
+struct session {
+	bool used;
+	bool app;
+	struct {
+		bool login;
+		bool mixed;
+	} tls;
+	time_t login_at;
+	time_t valid_until;
+	char remote_addr[48]; // Large enough for IPv4 and IPv6 addresses, hard-coded in civetweb.h as mg_request_info.remote_addr
+	char user_agent[128];
+	char sid[SID_SIZE];
+	char csrf[SID_SIZE];
+};
+
+#endif // AUTH_H

src/api/dhcp.c

@@ -0,0 +1,113 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2023 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  API Implementation /api/dhcp
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#include "FTL.h"
+#include "webserver/http-common.h"
+#include "webserver/json_macros.h"
+#include "api.h"
+#include "config/dnsmasq_config.h"
+// rotate_files()
+#include "files.h"
+
+int api_dhcp_leases_GET(struct ftl_conn *api)
+{
+	// Get DHCP leases
+	cJSON *leases = JSON_NEW_ARRAY();
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_ADD_ITEM_TO_OBJECT(json, "leases", leases);
+
+	FILE *fp = fopen(DHCPLEASESFILE, "r");
+	if(fp == NULL)
+	{
+		// File does not exist or not readable, send empty array
+		JSON_SEND_OBJECT(json);
+	}
+
+	char *line = NULL;
+	size_t len = 0;
+	ssize_t read;
+	while((read = getline(&line, &len, fp)) != -1)
+	{
+		// Skip empty lines
+		if(read == 0)
+			continue;
+
+		// Skip duid line
+		if(strncmp(line, "duid", 4) == 0)
+			continue;
+
+		// Parse line
+		unsigned long expires = 0;
+		char hwaddr[18] = { 0 };
+		char ip[INET_ADDRSTRLEN] = { 0 };
+		char name[65] = { 0 };
+		char clientid[765] = { 0 };
+		const int ret = sscanf(line, "%lu %17s %15s %64s %764s", &expires, hwaddr, ip, name, clientid);
+
+		// Skip invalid lines
+		if(ret != 5)
+			continue;
+
+		// Create JSON object for this lease
+		cJSON *lease = JSON_NEW_OBJECT();
+		JSON_ADD_NUMBER_TO_OBJECT(lease, "expires", expires);
+		JSON_COPY_STR_TO_OBJECT(lease, "hwaddr", hwaddr);
+		JSON_COPY_STR_TO_OBJECT(lease, "ip", ip);
+		JSON_COPY_STR_TO_OBJECT(lease, "name", name);
+		JSON_COPY_STR_TO_OBJECT(lease, "clientid", clientid);
+
+		// Add lease to array
+		JSON_ADD_ITEM_TO_ARRAY(leases, lease);
+	}
+	free(line);
+	fclose(fp);
+
+	JSON_SEND_OBJECT(json);
+}
+
+// defined in dnsmasq_interface.c
+extern bool FTL_unlink_DHCP_lease(const char *ipaddr, const char **hint);
+
+// Delete DHCP leases
+int api_dhcp_leases_DELETE(struct ftl_conn *api)
+{
+	// Validate input (must be a valid IPv4 address)
+	struct sockaddr_in sa;
+	if(api->item == NULL || strlen(api->item) == 0 || inet_pton(AF_INET, api->item, &(sa.sin_addr)) == 0)
+	{
+		// Send empty reply with code 204 No Content
+		return send_json_error(api,
+		                       400,
+		                       "bad_request",
+		                       "The provided IPv4 address is invalid",
+		                       api->item);
+	}
+
+	// Delete lease
+	log_debug(DEBUG_API, "Deleting DHCP lease for address %s", api->item);
+
+	const char *hint = NULL;
+	const bool found = FTL_unlink_DHCP_lease(api->item, &hint);
+	if(!found && hint != NULL)
+	{
+		// Send error when something went wrong (hint is not NULL)
+		return send_json_error(api,
+		                       400,
+		                       "bad_request",
+		                       "Failed to delete DHCP lease",
+		                       hint);
+	}
+
+	// Send empty reply with codes:
+	// - 204 No Content (if a lease was deleted)
+	// - 404 Not Found (if no lease was found)
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_SEND_OBJECT_CODE(json, found ? 204 : 404);
+}

src/api/dns.c

@@ -0,0 +1,143 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2019 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  API Implementation /api/dns
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#include "FTL.h"
+#include "webserver/http-common.h"
+#include "webserver/json_macros.h"
+#include "api.h"
+// {s,g}et_blockingstatus()
+#include "config/setupVars.h"
+// set_blockingmode_timer()
+#include "timers.h"
+#include "shmem.h"
+// config struct
+#include "config/config.h"
+
+// Location of custom.list
+#include "config/dnsmasq_config.h"
+
+#define DOMAIN_VALIDATION_REGEX "^((-|_)*[a-z0-9]((-|_)*[a-z0-9])*(-|_)*)(\\.(-|_)*([a-z0-9]((-|_)*[a-z0-9])*))*$"
+#define LABEL_VALIDATION_REGEX "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$"
+
+static int get_blocking(struct ftl_conn *api)
+{
+	// Return current status
+	cJSON *json = JSON_NEW_OBJECT();
+	const enum blocking_status blocking = get_blockingstatus();
+	switch(blocking)
+	{
+		case BLOCKING_ENABLED:
+			JSON_REF_STR_IN_OBJECT(json, "blocking", "enabled");
+			break;
+		case BLOCKING_DISABLED:
+			JSON_REF_STR_IN_OBJECT(json, "blocking", "disabled");
+			break;
+		case DNS_FAILED:
+			JSON_REF_STR_IN_OBJECT(json, "blocking", "failure");
+			break;
+		case BLOCKING_UNKNOWN:
+			JSON_REF_STR_IN_OBJECT(json, "blocking", "unknown");
+			break;
+	}
+
+	// Get timer information (if applicable)
+	double delay;
+	bool target_status;
+	get_blockingmode_timer(&delay, &target_status);
+	if(delay > -1)
+	{
+		JSON_ADD_NUMBER_TO_OBJECT(json, "timer", delay);
+	}
+	else
+	{
+		JSON_ADD_NULL_TO_OBJECT(json, "timer");
+	}
+
+	// Send object (HTTP 200 OK)
+	JSON_SEND_OBJECT(json);
+}
+
+static int set_blocking(struct ftl_conn *api)
+{
+	if(get_blockingstatus() == DNS_FAILED)
+	{
+		return send_json_error(api, 500,
+		                       "dns_failure",
+		                       "DNS resolver is not running",
+		                       NULL);
+	}
+
+	// Check if the payload is valid JSON
+	const int ret = check_json_payload(api);
+	if(ret != 0)
+		return ret;
+
+	cJSON *elem = cJSON_GetObjectItemCaseSensitive(api->payload.json, "blocking");
+	if (!cJSON_IsBool(elem))
+	{
+		return send_json_error(api, 400,
+		                       "body_error",
+		                       "No \"blocking\" boolean in body data",
+		                       NULL);
+	}
+	const enum blocking_status target_status = cJSON_IsTrue(elem) ? BLOCKING_ENABLED : BLOCKING_DISABLED;
+
+	// Get (optional) timer
+	double timer = -1;
+	elem = cJSON_GetObjectItemCaseSensitive(api->payload.json, "timer");
+	if (cJSON_IsNumber(elem) && elem->valuedouble > 0.0)
+		timer = elem->valuedouble;
+
+	if(target_status == get_blockingstatus())
+	{
+		// The blocking status does not need to be changed
+
+		// Delete a possibly running timer
+		set_blockingmode_timer(-1.0, true);
+
+		log_debug(DEBUG_API, "No change in blocking mode, resetting timer");
+	}
+	else
+	{
+		// Activate requested status
+		set_blockingstatus(target_status);
+
+		// Start timer (-1 disables all running timers)
+		set_blockingmode_timer(timer, !target_status);
+
+		log_debug(DEBUG_API, "%sd Pi-hole, timer set to %f seconds", target_status ? "Enable" : "Disable", timer);
+	}
+
+	// Return GET property as result of POST/PUT/PATCH action
+	// if no error happened above
+	return get_blocking(api);
+}
+
+int api_dns_blocking(struct ftl_conn *api)
+{
+	if(api->method == HTTP_GET)
+	{
+		lock_shm();
+		const int ret = get_blocking(api);
+		unlock_shm();
+		return ret;
+	}
+	else if(api->method == HTTP_POST)
+	{
+		lock_shm();
+		const int ret = set_blocking(api);
+		unlock_shm();
+		return ret;
+	}
+	else
+	{
+		return send_json_error(api, 405, "method_not_allowed", "Method not allowed", NULL);
+	}
+}

src/api/docs/CMakeLists.txt

@@ -0,0 +1,66 @@
+# Pi-hole: A black hole for Internet advertisements
+# (c) 2021 Pi-hole, LLC (https://pi-hole.net)
+# Network-wide ad blocking via your own hardware.
+#
+# FTL Engine
+# /src/api/docs/CMakeList.txt
+#
+# This file is copyright under the latest version of the EUPL.
+# Please see LICENSE file for your rights under this license.
+
+set(sources
+        hex/index.html
+        hex/index.css
+        hex/pi-hole.js
+        hex/external/rapidoc-min.js
+        hex/external/rapidoc-min.js.map
+        hex/external/highlight.min.js
+        hex/external/highlight-default.min.css
+        hex/images/logo.svg
+        hex/specs/action.yaml
+        hex/specs/auth.yaml
+        hex/specs/clients.yaml
+        hex/specs/config.yaml
+        hex/specs/common.yaml
+        hex/specs/dhcp.yaml
+        hex/specs/dns.yaml
+        hex/specs/docs.yaml
+        hex/specs/domains.yaml
+        hex/specs/endpoints.yaml
+        hex/specs/groups.yaml
+        hex/specs/history.yaml
+        hex/specs/info.yaml
+        hex/specs/lists.yaml
+        hex/specs/logs.yaml
+        hex/specs/main.yaml
+        hex/specs/network.yaml
+        hex/specs/queries.yaml
+        hex/specs/search.yaml
+        hex/specs/stats.yaml
+        hex/specs/teleporter.yaml
+        docs.c
+        )
+
+# Create relevant directories for processed content
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/hex)
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/hex/specs)
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/hex/images)
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/hex/external)
+
+# Compile files from content/ into hex/
+find_program(RESOURCE_COMPILER xxd)
+file(GLOB_RECURSE COMPILED_RESOURCES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/content" "content/*")
+foreach(INPUT_FILE ${COMPILED_RESOURCES})
+    set(IN ${CMAKE_CURRENT_SOURCE_DIR}/content/${INPUT_FILE})
+    set(OUTPUT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/hex/${INPUT_FILE})
+    add_custom_command(
+        OUTPUT hex/${INPUT_FILE}
+        COMMAND ${RESOURCE_COMPILER} -i < ${IN} > ${OUTPUT_FILE}
+        COMMENT "Compiling ${INPUT_FILE}"
+        VERBATIM)
+    list(APPEND COMPILED_RESOURCES ${OUTPUT_FILE})
+endforeach()
+
+add_library(api_docs OBJECT ${sources})
+target_compile_options(api_docs PRIVATE ${EXTRAWARN})
+target_include_directories(api_docs PRIVATE ${PROJECT_SOURCE_DIR}/src)

src/api/docs/content/external/highlight-default.min.css

@@ -0,0 +1 @@
+.hljs{display:block;overflow-x:auto;padding:.5em;background:#F0F0F0}.hljs,.hljs-subst{color:#444}.hljs-comment{color:#888888}.hljs-keyword,.hljs-attribute,.hljs-selector-tag,.hljs-meta-keyword,.hljs-doctag,.hljs-name{font-weight:bold}.hljs-type,.hljs-string,.hljs-number,.hljs-selector-id,.hljs-selector-class,.hljs-quote,.hljs-template-tag,.hljs-deletion{color:#880000}.hljs-title,.hljs-section{color:#880000;font-weight:bold}.hljs-regexp,.hljs-symbol,.hljs-variable,.hljs-template-variable,.hljs-link,.hljs-selector-attr,.hljs-selector-pseudo{color:#BC6060}.hljs-literal{color:#78A960}.hljs-built_in,.hljs-bullet,.hljs-code,.hljs-addition{color:#397300}.hljs-meta{color:#1f7199}.hljs-meta-string{color:#4d99bf}.hljs-emphasis{font-style:italic}.hljs-strong{font-weight:bold}

src/api/docs/content/images/logo.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 88.32 129.93"><defs><linearGradient id="New_Gradient_Swatch_1" x1="2.71" x2="69.77" y1="20.04" y2="20.04" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#12b212"/><stop offset="1" stop-color="#0f0"/></linearGradient><style>.cls-2{fill:#980200}.cls-3{fill:red}</style></defs><title>NewVortex</title><path fill="url(#New_Gradient_Swatch_1)" d="M36.56 39.93C20.34 38.2 4 25.94 2.71 0c25.17 0 38.63 14.9 39.93 38.51 4.76-28.32 27.07-25 27.07-25 1.06 16.05-12.12 25.78-27.07 26.59-4.2-8.85-29.36-30.56-29.36-30.56a.07.07 0 00-.11.08s24.28 21.15 23.39 30.31"/><path d="M44.16 129.93c-1.57-.09-16.22-.65-17.11-17.11-.72-10 7.18-17.37 7.18-27.08C32.44 61.53 0 64.53 0 85.74a19.94 19.94 0 005.83 14.14L30 124.06a19.94 19.94 0 0014.14 5.83" class="cls-2"/><path d="M88.32 85.75c-.09 1.57-.65 16.22-17.11 17.11-10 .72-17.38-7.18-27.08-7.18-24.21 1.79-21.21 34.22 0 34.22a19.94 19.94 0 0014.14-5.83L82.46 99.9a19.94 19.94 0 005.83-14.14" class="cls-3"/><path d="M44.16 41.59c1.57.09 16.22.65 17.11 17.11.72 10-7.18 17.37-7.18 27.08 1.79 24.21 34.22 21.21 34.22 0a19.94 19.94 0 00-5.83-14.14L58.3 47.45a19.94 19.94 0 00-14.14-5.83" class="cls-2"/><path d="M.08 85.75c.09-1.57.65-16.22 17.11-17.11 10-.72 17.38 7.18 27.08 7.18 24.21-1.82 21.21-34.22 0-34.22a19.94 19.94 0 00-14.14 5.83L5.94 71.61A19.94 19.94 0 00.11 85.75" class="cls-3"/></svg>

src/api/docs/content/index.css

@@ -0,0 +1,73 @@
+
+.btn {
+  padding: 6px 12px;
+  font-size:13px;
+  background-color: var(--primary-color);
+  color: #fff;
+  border: none;
+  margin: 0 2px;
+  border-radius: 2px;
+  cursor:pointer;
+}
+.btn.large {
+  width: 120px;
+  height: 24px
+}
+.btn.medium {
+  width: 75px;
+  height: 24px
+}
+.btn.small {
+  width: 60px;
+  height: 24px
+}
+.btn.green {
+  background-color: #228b22;
+}
+.btn.red {
+  background-color: #ff4500;
+}
+
+.btn:hover {
+  background: #fff;
+  color: var(--primary-color);
+}
+
+.logo-slot {
+  padding: 10px 0 10px 16px;
+  width: 50px;
+}
+
+.header-slot {
+  margin: 0 15px 0 0;
+  width: 100%;
+  display: flex;
+  gap: 6px 32px;
+  flex-wrap: wrap;
+  justify-content: start;
+}
+.header-title {
+  font-weight: 700;
+  font-size: 32px;
+  line-height: 1;
+}
+
+.password-controls {
+  display: flex;
+  align-items: stretch;
+}
+
+.footer-slot {
+  margin:0;
+  padding:16px;
+  display: flex;
+  gap: 16px 36px;
+  flex-wrap: wrap;
+  justify-content: space-between;
+  color:#fff;
+  text-align:center;
+  background-color:#222;
+}
+.footer-slot > div {
+  flex: 0 0 auto;
+}

src/api/docs/content/index.html

@@ -0,0 +1,67 @@
+<!doctype html>
+<html lang="en">
+    <head>
+        <meta charset="utf-8">
+        <meta name="viewport" content="width=device-width,minimum-scale=1,initial-scale=1,user-scalable=yes">
+        <title>Pi-hole API documentation</title>
+        <!--<link href="https://fonts.googleapis.com/css?family=Open+Sans:300,600&display=swap" rel="stylesheet">-->
+        <link rel="stylesheet" href="external/highlight-default.min.css">
+        <script src="external/highlight.min.js"></script>
+        <script type='text/javascript' src='external/rapidoc-min.js'></script>
+        <script type="text/javascript" src="pi-hole.js"></script>
+        <link href='index.css' rel='stylesheet'>
+        <link rel="apple-touch-icon" href="/admin/img/favicons/apple-touch-icon.png" sizes="180x180">
+        <link rel="icon" href="/admin/img/favicons/favicon-32x32.png" sizes="32x32" type="image/png">
+        <link rel="icon" href="/admin/img/favicons/favicon-16x16.png" sizes="16x16" type="image/png">
+        <link rel="manifest" href="/admin/img/favicons/manifest.json">
+        <link rel="mask-icon" href="/admin/img/favicons/safari-pinned-tab.svg" color="#367fa9">
+        <link rel="shortcut icon" href="/admin/img/favicons/favicon.ico">
+        <meta name="msapplication-TileColor" content="#367fa9">
+        <meta name="msapplication-TileImage" content="/admin/img/favicons/mstile-150x150.png">
+        <meta name="theme-color" content="#367fa9">
+    </head>
+    <body>
+        <rapi-doc id = "thedoc"
+          spec-url = "specs/main.yaml"
+          allow-server-selection = "true"
+          allow-authentication = "false"
+          allow-spec-url-load = "false"
+          allow-spec-file-load = "false"
+          show-header = "true"
+          show-info = "true"
+          theme = "dark"
+          allow-try = "true"
+          sort-endpoints-by = "path"
+          default-schema-tab = "model"
+          schema-style = "tree"
+          render-style = "view"
+          primary-color = "#2d87e2"
+          header-color = "#222"
+          api-key-name = "sid"
+          api-key-location = "header"
+          api-key-value = "-"
+          show-method-in-nav-bar="as-colored-block"
+          allow-search = "false">
+            <img slot="logo" class="logo-slot" src="images/logo.svg">
+            <div slot="header" class="header-slot">
+                <div class="header-title">Pi-hole API Documentation</div>
+                <div class="password-controls">
+                    <input id="loginpw" type="password" placeholder="password">
+                    <button class="btn" id="loginbtn" onclick="loginout()">Login</button>
+                </div>
+            </div>
+            <!-- content at the bottom -->
+            <div slot="footer" class="footer-slot">
+                <div>
+                    <button onclick="document.getElementById('thedoc').setAttribute('theme', 'dark')" class="btn">Dark Theme</button>
+                    <button onclick="document.getElementById('thedoc').setAttribute('theme', 'light')" class="btn">Light Theme</button>
+                </div>
+                <div>
+                    <button onclick="setStyle('view')" class="btn">Default</button>
+                    <button onclick="setStyle('read')" class="btn">Reader</button>
+                    <button onclick="setStyle('focused')" class="btn">Focused reader</button>
+                </div>
+            </div>
+        </rapi-doc>
+    </body>
+</html>

src/api/docs/content/pi-hole.js

@@ -0,0 +1,113 @@
+/* Pi-hole: A black hole for Internet advertisements
+ *  (c) 2021 Pi-hole, LLC (https://pi-hole.net)
+ *  Network-wide ad blocking via your own hardware.
+ *
+ *  This file is copyright under the latest version of the EUPL.
+ *  Please see LICENSE file for your rights under this license. */
+
+// GET implementation
+async function getData(url = '') {
+	const docEl = document.getElementById('thedoc');
+	const sid = docEl.attributes["api-key-value"].value;
+	const response = await fetch(url, {
+		method: 'GET',
+		headers: {'Content-Type': 'application/json', 'X-FTL-SID': sid}
+	});
+	return response.json();
+}
+
+// DELETE implementation
+async function deleteData(url = '') {
+	const docEl = document.getElementById('thedoc');
+	const sid = docEl.attributes["api-key-value"].value;
+	const response = await fetch(url, {
+		method: 'DELETE',
+		headers: {'X-FTL-SID': sid}
+	});
+	return response;
+}
+
+// POST implementation
+async function postData(url = '', data = {}) {
+	const docEl = document.getElementById('thedoc');
+	const sid = docEl.attributes["api-key-value"].value;
+	const response = await fetch(url, {
+		method: 'POST',
+		headers: {'Content-Type': 'application/json', 'X-FTL-SID': sid},
+		body: JSON.stringify(data)
+	});
+	return response.json();
+}
+
+// Mark login as OK
+function loginOk(sid) {
+	const docEl = document.getElementById('thedoc');
+	docEl.setAttribute('api-key-value', sid);
+	const btn = document.getElementById('loginbtn');
+	btn.classList.add('green');
+	btn.classList.remove('red');
+	btn.textContent = 'Logout';
+}
+
+// Mark login as FAIL
+function loginFAIL() {
+	const docEl = document.getElementById('thedoc');
+	docEl.setAttribute('api-key-value', '-');
+	const btn = document.getElementById('loginbtn');
+	btn.classList.remove('green');
+	btn.classList.add('red');
+	btn.textContent = 'Login';
+}
+
+// Mark logout as OK
+function logoutOk() {
+	const docEl = document.getElementById('thedoc');
+	docEl.setAttribute('api-key-value', "-");
+	const btn = document.getElementById('loginbtn');
+	btn.classList.remove('green');
+	btn.classList.remove('red');
+	btn.textContent = 'Login';
+}
+
+// Login using password
+function loginout(){
+	const docEl = document.getElementById('thedoc');
+	if(docEl.attributes["api-key-value"].value === '-') {
+		var pw = document.getElementById('loginpw').value;
+		postData('/api/auth', {password: pw})
+		.then(data => {
+			if(data.session.valid === true) {
+				loginOk(data.session.sid);
+			} else {
+				loginFAIL();
+			}
+		})
+		.catch((error) => {
+			loginFAIL();
+			console.error('Error:', error);
+		});
+	} else {
+		deleteData('/api/auth')
+		.then(logoutOk())
+		.catch((error) => {
+			loginFAIL();
+			console.error('Error:', error);
+		});
+	}
+}
+
+function setStyle(style) {
+	const docEl = document.getElementById('thedoc');
+	docEl.setAttribute('render-style', style);
+	docEl.setAttribute('allow-search', style !== 'view');
+}
+document.addEventListener('DOMContentLoaded', (event) => {
+	let docEl = document.getElementById("thedoc");
+
+	docEl.addEventListener('after-try', (e) => {
+		console.log(e.detail.response);
+		if(e.detail.response.status === 401) {
+			loginFAIL();
+		}
+	});
+});

src/api/docs/content/specs/action.yaml

@@ -0,0 +1,170 @@
+openapi: 3.0.2
+components:
+  paths:
+    gravity:
+      post:
+        summary: Run gravity
+        tags:
+          - Actions
+        operationId: "action_gravity"
+        description: |
+          Update Pi-hole's adlists by running `pihole -g`. The output of the process is streamed with chunked encoding.
+        responses:
+          '200':
+            description: OK
+            content:
+              text/plain:
+                schema:
+                  type: string
+                  example: |
+                      [i] Neutrino emissions detected...
+
+                      [✓] Pulling blocklist source list into range
+
+                      [i] Preparing new gravity database...
+                      [✓] Preparing new gravity database
+                      [i] Using libz compression
+
+                      [i] Target: https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts
+                      [✓] Status: Retrieval successful
+                      [i] Imported 172502 domains, ignoring 3 non-domain entries
+                          Sample of non-domain entries:
+                            - 0.0.0.0
+                            - fe
+                            - ff
+                      [i] List stayed unchanged
+
+                      [i] Target: https://v.firebog.net/hosts/AdguardDNS.txt
+                      [✓] Status: No changes detected
+                      [i] Imported 47225 domains
+
+                      [✓] Creating new gravity databases
+                      [✓] Storing downloaded domains in new gravity database
+                      [✓] Building tree
+                      [✓] Swapping databases
+                      [✓] The old database remains available.
+                      [i] Number of gravity domains: 219727 (215440 unique domains)
+                      [i] Number of exact blacklisted domains: 0
+                      [i] Number of regex blacklist filters: 2
+                      [i] Number of exact whitelisted domains: 0
+                      [i] Number of regex whitelist filters: 0
+                      [✓] Cleaning up stray matter
+
+                      [✓] FTL is listening on port
+                         [✓] UDP (IPv4)
+                         [✓] TCP (IPv4)
+                         [✓] UDP (IPv6)
+                         [✓] TCP (IPv6)
+
+                      [✓] Pi-hole blocking is enabled
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/errors/unauthorized'
+    restartdns:
+      post:
+        summary: Restart pihole-FTL
+        tags:
+          - Actions
+        operationId: "action_restartdns"
+        description: |
+          Restarts the pihole-FTL service
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/schemas/success'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    flush_logs:
+      post:
+        summary: Flush the DNS logs
+        tags:
+          - Actions
+        operationId: "action_flushlogs"
+        description: |
+          Flushes the DNS logs. This includes emptying the DNS log file and purging the most recent 24 hours from both the database and FTL's internal memory.
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/schemas/success'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    flush_arp:
+      post:
+        summary: Flush the network table
+        tags:
+          - Actions
+        operationId: "action_flusharp"
+        description: |
+          Flushes the network table. This includes emptying the ARP table and removing both all known devices and their associated addresses.
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/schemas/success'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '403':
+            description: Forbidden
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'action.yaml#/components/errors/forbidden'
+
+  errors:
+    forbidden:
+      description: |
+        The request was valid, but the server is refusing the action.
+      type: object
+      properties:
+        error:
+          type: object
+          properties:
+            key:
+              type: string
+              description: "Machine-readable error type"
+              example: "forbidden"
+            message:
+              type: string
+              description: "Human-readable error message"
+              example: "<Action name> is not allowed"
+            hint:
+              type: string
+              nullable: true
+              description: "No additional data available"
+              example: "Check setting webserver.api.allow_destructive"

src/api/docs/content/specs/auth.yaml

@@ -0,0 +1,584 @@
+openapi: 3.0.2
+components:
+  paths:
+    auth:
+      get:
+        summary: Check if authentication is required
+        tags:
+          - Authentication
+        operationId: "get_auth"
+        security: []
+        description: |
+          The API may chose to reply with a valid session if no authentication is needed for this server.
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'auth.yaml#/components/schemas/session'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  auth_okay:
+                    $ref: 'auth.yaml#/components/examples/auth_okay'
+                  no_login_required:
+                    $ref: 'auth.yaml#/components/examples/no_login_required'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'auth.yaml#/components/schemas/session'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  login_required:
+                    $ref: 'auth.yaml#/components/examples/login_required'
+                  login_required_2fa:
+                    $ref: 'auth.yaml#/components/examples/login_required_2fa'
+      post:
+        summary: Submit password for login
+        tags:
+          - Authentication
+        operationId: "add_auth"
+        security: []
+        description: |
+          Authenticate using a password. The password isn't stored in the session nor used to create the session token. Instead, the session token is produced using a cryptographically secure random number generator. A CSRF token is utilized to guard against CSRF attacks and is necessary when using Cookie-based authentication. However, it's not needed with other authentication methods.
+
+          Both the Session ID (SID) and CSRF token remain valid for the session's duration. The session can be extended before its expiration by performing any authenticated action. By default, the session lasts for 5 minutes. It can be invalidated by either logging out or deleting the session. Additionally, the session becomes invalid when the password is altered or a new application password is created.
+
+          If two-factor authentication (2FA) is activated, the Time-based One-Time Password (TOTP) token must be included in the request body. Be aware that the TOTP token, generated by your authenticator app, is only valid for 30 seconds. If the TOTP token is missing, invalid, or has been used previously, the login attempt will be unsuccessful.
+        requestBody:
+          description: Callback payload
+          content:
+            'application/json':
+              schema:
+                $ref: 'auth.yaml#/components/schemas/password'
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'auth.yaml#/components/schemas/session'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  login_okay:
+                    $ref: 'auth.yaml#/components/examples/login_okay'
+                  no_login_required:
+                    $ref: 'auth.yaml#/components/examples/no_login_required'
+          '400':
+            description: Bad Request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'auth.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  no_payload:
+                    $ref: 'auth.yaml#/components/examples/errors/no_payload'
+                  no_password:
+                    $ref: 'auth.yaml#/components/examples/errors/no_password'
+                  password_inval:
+                    $ref: 'auth.yaml#/components/examples/errors/password_inval'
+                  totp_missing:
+                    $ref: 'auth.yaml#/components/examples/errors/totp_missing'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  totp_invalid:
+                    $ref: 'auth.yaml#/components/examples/errors/totp_invalid'
+                  totp_reused:
+                    $ref: 'auth.yaml#/components/examples/errors/totp_reused'
+          '429':
+            description: Too Many Requests
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/too_many_requests'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  rate_limit:
+                    $ref: 'auth.yaml#/components/examples/errors/rate_limit'
+                  no_seats:
+                    $ref: 'auth.yaml#/components/examples/errors/no_seats'
+      delete:
+        summary: Delete session
+        tags:
+          - Authentication
+        operationId: "delete_groups"
+        description: |
+          This endpoint can be used to delete the current session. It will
+          invalidate the session token and the CSRF token. The session can be
+          extended before its expiration by performing any authenticated action.
+          By default, the session lasts for 5 minutes. It can be invalidated by
+          either logging out or deleting the session. Additionally, the session
+          becomes invalid when the password is altered or a new application
+          password is created.
+
+          You can also delete a session by its ID using the `DELETE /auth/session/{id}` endpoint.
+
+          Note that you cannot delete the current session if you have not
+          authenticated (e.g., no password has been set on your Pi-hole).
+        responses:
+          '204':
+            description: No Content (deleted)
+          '404':
+            description: Not Found (no session active)
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    session_list:
+      get:
+        summary: List of all current sessions
+        tags:
+          - Authentication
+        operationId: "get_auth_sessions"
+        description: List of all current sessions including their validity and further information about the client such as the IP address and user agent.
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'auth.yaml#/components/schemas/sessions_list'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    totp:
+      get:
+        summary: Suggest new TOTP credentials
+        tags:
+          - Authentication
+        operationId: "get_auth_totp"
+        description: Suggest new TOTP credentials for two-factor authentication (2FA)
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'auth.yaml#/components/schemas/totp'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    session:
+      delete:
+        summary: Delete session by ID
+        parameters:
+          - $ref: 'auth.yaml#/components/parameters/id'
+        tags:
+          - Authentication
+        operationId: "delete_auth_session"
+        description: |
+          Using this endpoint, a session can be deleted by its ID.
+        responses:
+          '204':
+            description: No Content (deleted)
+          '404':
+            description: Not Found (session not found)
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
+          '400':
+            description: Bad Request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'auth.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  missing_session_id:
+                    $ref: 'auth.yaml#/components/examples/errors/missing_session_id'
+                  session_id_oob:
+                    $ref: 'auth.yaml#/components/examples/errors/no_password'
+                  session_not_in_use:
+                    $ref: 'auth.yaml#/components/examples/errors/session_not_in_use'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    app:
+      get:
+        summary: Create new application password
+        tags:
+          - Authentication
+        operationId: "add_app"
+        description: |
+          Create a new application password. The generated password is shown only once and cannot be retrieved later - make sure to store it in a safe place. The application password can be used to authenticate against the API instead of the regular password.
+          It does not require 2FA verification. Generating a new application password will invalidate all currently active sessions.
+
+          Note that this endpoint only generates an application password accompanied by its hash. To make this new password effective, the returned `hash` has to be set as `webserver.api.app_password` in the Pi-hole configuration in a follow-up step. This can be done in various ways, e.g. via the API (`PATCH /api/config/webserver/api/app_pwhash`), the graphical web interface (Settings -> All Settings) or by editing the configuration file directly.
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'auth.yaml#/components/schemas/app'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+
+  schemas:
+    session:
+      type: object
+      required:
+        - session
+      properties:
+        session:
+          type: object
+          description: Session object
+          required:
+            - valid
+            - sid
+            - csrf
+            - validity
+            - message
+            - totp
+          properties:
+            valid:
+              type: boolean
+              description: Valid session indicator (client is authenticated)
+            totp:
+              type: boolean
+              description: Whether 2FA (TOTP) is enabled on this Pi-hole
+            sid:
+              type: string
+              description: Session ID
+              nullable: true
+            csrf:
+              type: string
+              description: CSRF token
+              nullable: true
+            validity:
+              type: integer
+              description: Remaining lifetime of this session unless refreshed (seconds)
+            message:
+              type: string
+              description: Human-readable message describing the session status
+              nullable: true
+
+    password:
+      type: object
+      description: Password to be sent to the API
+      properties:
+        password:
+          type: string
+          description: Password to be used for login
+          example: abcdef
+    sessions_list:
+      type: object
+      description: List of all current sessions
+      properties:
+        sessions:
+          type: array
+          items:
+            type: object
+            description: Session object
+            properties:
+              id:
+                type: number
+                description: Session ID
+              current_session:
+                type: boolean
+                description: Indicator if this is the current session
+              valid:
+                type: boolean
+                description: Valid session indicator (existing sessions may be invalid due to timeout)
+              tls:
+                type: object
+                description: TLS (end-to-end encryption) information
+                properties:
+                  login:
+                    type: boolean
+                    description: Indicator if TLS (end-to-end encryption) has been used during login for this session
+                  mixed:
+                    type: boolean
+                    description: Indicator if TLS (end-to-end encryption) is used only partially for this session
+              app:
+                type: boolean
+                description: Indicator if this session was initiated using an application password
+              login_at:
+                type: integer
+                description: Timestamp of login (seconds since epoch)
+              last_active:
+                type: integer
+                description: Timestamp of last activity (seconds since epoch)
+              valid_until:
+                type: integer
+                description: Timestamp of session expiration (seconds since epoch)
+              remote_addr:
+                type: string
+                description: IP address of the client
+              user_agent:
+                type: string
+                description: User agent of the client
+          example:
+            - id: 1
+              current_session: true
+              valid: true
+              tls:
+                login: true
+                mixed: false
+              app: false
+              login_at: 1580000000
+              last_active: 1580000000
+              valid_until: 1580000300
+              remote_addr: "192.168.0.34"
+              user_agent: "Mozilla/5.0 (X11; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0"
+    totp:
+      type: object
+      description: TOTP secret suggestion
+      properties:
+        totp:
+          type: object
+          properties:
+            type:
+              type: string
+            account:
+              type: string
+            issuer:
+              type: string
+            algorithm:
+              type: string
+            digits:
+              type: integer
+            period:
+              type: integer
+            offset:
+              type: integer
+            secret:
+              type: string
+            codes:
+              type: array
+              items:
+                type: integer
+    app:
+      type: object
+      description: Application password
+      properties:
+        app:
+          type: object
+          properties:
+            password:
+              type: string
+            hash:
+              type: string
+          example:
+            password: "7ua0rHPZixX6B3bTa5o4Dv08iyEIm/2q9qgLrF9MNVw="
+            hash: "$BALLOON-SHA256$v=1$s=1024,t=32$4ERwJD4XucRP4PcDHNLiAg==$kSy0Eou8RUVtK2UTbc5MCpItV8YC3VRuoGhoENxSQ2I="
+
+  errors:
+    bad_request:
+      type: object
+      description: Bad request
+      properties:
+        error:
+          type: object
+          properties:
+            key:
+              type: string
+              description: "Machine-readable error type"
+            message:
+              type: string
+              description: "Human-readable error message"
+            hint:
+              type: string
+              description: "Additional data (if available)"
+              nullable: true
+
+  examples:
+    auth_okay:
+      summary: Session valid
+      value:
+        session:
+          valid: true
+          totp: false
+          sid: null
+          csrf: null
+          validity: 300
+          message: null
+    login_okay:
+      summary: Login successful
+      value:
+        session:
+          valid: true
+          totp: false
+          sid: "vFA+EP4MQ5JJvJg+3Q2Jnw="
+          csrf: "Ux87YTIiMOf/GKCefVIOMw="
+          validity: 300
+          message: correct password
+    no_login_required:
+      summary: No login required for this client
+      value:
+        session:
+          valid: true
+          totp: false
+          sid: null
+          csrf: null
+          validity: -1
+          message: no auth for local user
+    login_required:
+      summary: Login required, 2FA disabled
+      value:
+        session:
+          valid: false
+          totp: false
+          sid: null
+          csrf: null
+          validity: -1
+          message: password incorrect
+    login_required_2fa:
+      summary: Login required, 2FA enabled
+      value:
+        session:
+          valid: false
+          totp: true
+          sid: null
+          csrf: null
+          validity: -1
+          message: password incorrect
+    login_failed:
+      summary: Login failed
+      value:
+        session:
+          valid: false
+          totp: false
+          sid: null
+          csrf: null
+          validity: -1
+          message: no SID provided
+    errors:
+      no_payload:
+        summary: Bad request (no valid JSON payload)
+        value:
+          error:
+            key: "bad_request"
+            message: "No valid JSON payload found"
+            hint: null
+      no_password:
+        summary: Bad request (no password)
+        value:
+          error:
+            key: "bad_request"
+            message: "No password found in JSON payload"
+            hint: null
+      password_inval:
+        summary: Bad request (password is not a string)
+        value:
+          error:
+            key: "bad_request"
+            message: "Field password has to be of type 'string'"
+            hint: null
+      totp_missing:
+        summary: Bad request (2FA token missing)
+        value:
+          error:
+            key: "bad_request"
+            message: "No 2FA token found in JSON payload"
+            hint: null
+      missing_session_id:
+        summary: Bad request (missing session ID)
+        value:
+          error:
+            key: "bad_request"
+            message: "Missing or invalid session ID"
+            hint: null
+      session_id_oob:
+        summary: Bad request (session ID out of bounds)
+        value:
+          error:
+            key: "bad_request"
+            message: "Session ID out of bounds"
+            hint: null
+      session_not_in_use:
+        summary: Bad request (session not in use)
+        value:
+          error:
+            key: "bad_request"
+            message: "Session ID not in use"
+            hint: null
+      totp_invalid:
+        summary: 2FA token invalid
+        value:
+          error:
+            key: "unauthorized"
+            message: "Invalid 2FA token"
+            hint: null
+      totp_reused:
+        summary: 2FA token reused
+        value:
+          error:
+            key: "unauthorized"
+            message: "Reused 2FA token"
+            hint: "wait for new token"
+      rate_limit:
+        summary: Rate limit exceeded
+        value:
+          error:
+            key: "rate_limiting"
+            message: "Rate-limiting login attempts"
+            hint: null
+      no_seats:
+        summary: No free API seats available
+        value:
+          error:
+            key: "api_seats_exceeded"
+            message: "API seats exceeded"
+            hint: "increase webserver.api.max_sessions"
+  parameters:
+    id:
+      in: path
+      name: id
+      schema:
+        type: integer
+      required: true
+      description: Session ID
+      example: 0

src/api/docs/content/specs/clients.yaml

@@ -0,0 +1,489 @@
+openapi: 3.0.2
+components:
+  paths:
+    client:
+      summary: Modify client
+      parameters:
+        - $ref: 'clients.yaml#/components/parameters/client'
+      get:
+        summary: Get clients
+        tags:
+          - "Client management"
+        operationId: "get_clients"
+        description: |
+          `{client}` is optional. If it is specified, it will result in only the requested client being returned. This parameter needs to be URI-encoded.
+
+          Valid combinations are:
+          - `/api/clients` (all clients)
+          - `/api/clients/my_client` (client identical to `my_client`)
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'clients.yaml#/components/schemas/clients/get'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  clients:
+                    $ref: 'clients.yaml#/components/examples/clients'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+      put:
+        summary: Replace client
+        tags:
+          - "Client management"
+        operationId: "replace_client"
+        description: |
+          Items may be updated by replacing them. `{client}` is required and needs to be URI-encoded.
+
+          Ensure to send all the required parameters (such as `comment` or `groups`) to ensure these properties are retained.
+          The read-only fields `id` and `date_added` are preserved, `date_modified` is automatically updated on success.
+        requestBody:
+          description: Callback payload
+          content:
+            application/json:
+              schema:
+                allOf:
+                  - $ref: 'clients.yaml#/components/schemas/clients/put'
+                  - $ref: 'common.yaml#/components/schemas/took'
+        responses:
+          '201':
+            description: Created item
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'clients.yaml#/components/schemas/clients/get' # identical to GET
+                    - $ref: 'clients.yaml#/components/schemas/lists_processed'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  item_missing:
+                    $ref: 'clients.yaml#/components/examples/errors/uri_error/item_missing'
+                  no_payload:
+                    $ref: 'clients.yaml#/components/examples/errors/bad_request/no_payload'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+      delete:
+        summary: Delete client
+        tags:
+          - "Client management"
+        operationId: "delete_client"
+        description: |
+          *Note:* There will be no content on success. `{client}` is required and needs to be URI-encoded.
+        responses:
+          '204':
+            description: Item deleted
+          '404':
+            description: Item not found
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  item_missing:
+                    $ref: 'clients.yaml#/components/examples/errors/uri_error/item_missing'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    suggestions:
+      get:
+        summary: Get client suggestions
+        tags:
+          - "Client management"
+        operationId: "get_client_suggestions"
+        description: |
+          Returns a list of unconfigured clients that have been seen by Pi-hole.
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'clients.yaml#/components/schemas/suggestions'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    direct:
+      post:
+        summary: Add new client
+        tags:
+          - "Client management"
+        operationId: "add_client"
+        description: |
+          Creates a new client in the `clients` object. The `{client}` itself is specified in the request body (POST JSON).
+
+          Clients may be described either by their IP addresses (IPv4 and IPv6 are supported),
+          IP subnets (CIDR notation, like `192.168.2.0/24`), their MAC addresses (like `12:34:56:78:9A:BC`), by their hostnames (like `localhost`), or by the interface they are connected to (prefaced with a colon, like `:eth0`).
+
+          Note that client recognition by IP addresses (incl. subnet ranges) is preferred over MAC address, host name or interface recognition as the two latter will only be available after some time.
+          Furthermore, MAC address recognition only works for devices at most one networking hop away from your Pi-hole.
+
+          On success, a new resource is created at `/clients/{client}`.
+
+          The `database_error` with message `UNIQUE constraint failed` error indicates that this client already exists.
+        requestBody:
+          description: Callback payload
+          content:
+            'application/json':
+              schema:
+                allOf:
+                  - $ref: 'clients.yaml#/components/schemas/clients/post'
+                  - $ref: 'common.yaml#/components/schemas/took'
+        responses:
+          '201':
+            description: Created item
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'clients.yaml#/components/schemas/clients/get' # identical to GET
+                    - $ref: 'clients.yaml#/components/schemas/lists_processed'
+                    - $ref: 'common.yaml#/components/schemas/took'
+            headers:
+              Location:
+                $ref: 'common.yaml#/components/headers/Location'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/schemas/took'
+                    - $ref: 'common.yaml#/components/errors/bad_request'
+                examples:
+                  no_payload:
+                    $ref: 'clients.yaml#/components/examples/errors/bad_request/no_payload'
+                  duplicate:
+                    $ref: 'clients.yaml#/components/examples/errors/database_error/duplicate'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/schemas/took'
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+    batchDelete:
+      post:
+        summary: Delete multiple clients
+        tags:
+          - "Client management"
+        operationId: "batchDelete_clients"
+        description: |
+          Deletes multiple clients in the `clients` object. The `{client}`s themselves are specified in the request body (POST JSON).
+
+          Clients may be described either by their IP addresses (IPv4 and IPv6 are supported),
+          IP subnets (CIDR notation, like `192.168.2.0/24`), their MAC addresses (like `12:34:56:78:9A:BC`), by their hostnames (like `localhost`), or by the interface they are connected to (prefaced with a colon, like `:eth0`).</p>
+
+          *Note:* There will be no content on success.
+        requestBody:
+          description: Callback payload
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    item:
+                      type: string
+                      description: client IP / MAC / hostname / interface
+                example:
+                  - "item": "192.168.2.5"
+                  - "item": "::1"
+                  - "item": "12:34:56:78:9A:BC"
+                  - "item": "localhost"
+                  - "item": ":eth0"
+        responses:
+          '204':
+            description: Items deleted
+          '404':
+            description: Item not found
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  no_payload:
+                    $ref: 'clients.yaml#/components/examples/errors/bad_request/no_payload'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+  schemas:
+    clients:
+      get:
+        type: object
+        properties:
+          clients:
+            type: array
+            description: Array of clients
+            items:
+              allOf:
+                - $ref: 'clients.yaml#/components/schemas/client_object'
+                - $ref: 'clients.yaml#/components/schemas/comment'
+                - $ref: 'clients.yaml#/components/schemas/groups'
+                - $ref: 'clients.yaml#/components/schemas/readonly'
+      post:
+        allOf:
+          - $ref: 'clients.yaml#/components/schemas/client_maybe_array'
+          - $ref: 'clients.yaml#/components/schemas/comment'
+          - $ref: 'clients.yaml#/components/schemas/groups'
+      put:
+        allOf:
+          - $ref: 'clients.yaml#/components/schemas/comment'
+          - $ref: 'clients.yaml#/components/schemas/groups'
+    suggestions:
+      type: object
+      properties:
+        clients:
+          type: array
+          items:
+            type: object
+            properties:
+              hwaddr:
+                type: string
+                nullable: true
+                example: "12:34:56:78:9A:BC"
+              macVendor:
+                type: string
+                nullable: true
+                example: "Espressif Inc."
+              lastQuery:
+                type: integer
+                example: 1683305917
+              addresses:
+                type: string
+                nullable: true
+                description: Comma-separated list of IP addresses
+                example: "127.0.0.1,::1"
+              names:
+                type: string
+                nullable: true
+                description: Comma-separated list of hostnames (if available)
+                example: "localhost,ip6-localhost"
+    client:
+      description: client IP / MAC / hostname / interface
+      type: string
+      example: 127.0.0.1
+    client_array:
+      description: array of client IPs / MACs / hostnames / interfaces
+      type: array
+      items:
+        type: string
+      example: ["127.0.0.1", "192.168.2.12"]
+    client_maybe_array:
+      type: object
+      properties:
+        client:
+          oneOf:
+            - $ref: 'clients.yaml#/components/schemas/client'
+            - $ref: 'clients.yaml#/components/schemas/client_array'
+    client_object:
+      type: object
+      properties:
+        client:
+          $ref: 'clients.yaml#/components/schemas/client'
+    comment:
+      type: object
+      properties:
+        comment:
+          description: User-provided free-text comment for this client
+          type: string
+          nullable: true
+          default: null
+          example: Some comment for this client
+    name:
+      type: object
+      properties:
+        name:
+          description: hostname (only available when {client} is an IP address)
+          type: string
+          readOnly: true
+          nullable: true
+          example: localhost
+    groups:
+      type: object
+      properties:
+        groups:
+          description: Array of group IDs
+          type: array
+          default: [0]
+          items:
+            type: integer
+    readonly:
+      type: object
+      properties:
+        id:
+          description: Database ID
+          type: integer
+          readOnly: true
+          example: 1
+        date_added:
+          description: Unix timestamp of item addition
+          type: integer
+          readOnly: true
+          example: 1611239095
+        date_modified:
+          description: Unix timestamp of last item modification
+          type: integer
+          readOnly: true
+          example: 1611239099
+        name:
+          description: hostname (only if available)
+          type: string
+          readOnly: true
+          nullable: true
+          example: localhost
+    lists_processed:
+      type: object
+      properties:
+        processed:
+          type: object
+          nullable: true
+          description: |
+            Object containing the number of clients that were successfully
+            added to the database and the number of clients that could not be
+            added to the database.
+          properties:
+            success:
+              description: |
+                Array of clients that were successfully added to the database.
+              type: array
+              items:
+                type: object
+                properties:
+                  item:
+                    description: Client that was added to the database
+                    type: string
+            errors:
+              description: |
+                Array of errors that occurred during processing.
+              type: array
+              items:
+                type: object
+                properties:
+                  item:
+                    description: Client that could not be added to the database
+                    type: string
+                  error:
+                    description: Error message
+                    type: string
+          example:
+            success:
+              - item: "127.0.0.1"
+              - item: "::1"
+            errors:
+              - item: "192.168.2.5"
+                error: "UNIQUE constraint failed: client.ip"
+  examples:
+    clients:
+      value:
+        clients:
+          - client: "127.0.0.1"
+            name: "localhost"
+            comment: "comment"
+            id: 1
+            date_added: 1604871899
+            date_modified: 1604871899
+          - client: "::1"
+            name: "ip6-localhost"
+            comment: null
+            id: 2
+            date_added: 1611322675
+            date_modified: 1611325497
+        took: 0.012
+        processed:
+          success:
+            - item: "127.0.0.1"
+          failed:
+            - item: "127.0.0.2"
+              error: "UNIQUE constraint failed: clientlist.client"
+    errors:
+      uri_error:
+        item_missing:
+          summary: Client to be modified is missing
+          value:
+            error:
+              key: "uri_error"
+              message: "Invalid request: Specify item in URI"
+              hint: null
+      bad_request:
+        no_payload:
+          summary: No JSON payload found
+          value:
+            error:
+              key: "bad_request"
+              message: "Invalid request body data (no valid JSON)"
+              hint: null
+      database_error:
+        duplicate:
+          summary: Database error
+          value:
+            error:
+              key: "database_error"
+              message: "Could not add to gravity database"
+              hint: "UNIQUE constraint failed: clientlist.client"
+  parameters:
+    client:
+      in: path
+      name: client
+      schema:
+        type: string
+      required: true
+      description: client IP / MAC / hostname / interface
+      example: 127.0.0.1

src/api/docs/content/specs/common.yaml

@@ -0,0 +1,106 @@
+openapi: 3.0.2
+components:
+  schemas:
+    took:
+      type: object
+      properties:
+        took:
+          type: number
+          description: Time in seconds it took to process the request
+          example: 0.003
+    success:
+      type: object
+      properties:
+        status:
+          type: string
+          description: Key indicating the status of the request
+          example: "success"
+  errors:
+    bad_request:
+      type: object
+      description: Bad request
+      properties:
+        error:
+          type: object
+          properties:
+            key:
+              type: string
+              description: "Machine-readable error type"
+            message:
+              type: string
+              description: "Human-readable error message"
+            hint:
+              type: string
+              description: "Further details"
+              nullable: true
+    unauthorized:
+      type: object
+      description: Authentication required
+      properties:
+        error:
+          type: object
+          properties:
+            key:
+              type: string
+              description: "Machine-readable error type"
+              example: "unauthorized"
+            message:
+              type: string
+              description: "Human-readable error message"
+              example: "Unauthorized"
+            hint:
+              type: string
+              nullable: true
+              description: "No additional data available"
+              example: null
+    too_many_requests:
+      type: object
+      description: "Too many requests (rate limiting)"
+      properties:
+        error:
+          type: object
+          properties:
+            key:
+              type: string
+              description: "Machine-readable error type"
+              example: "too_many_requests"
+            message:
+              type: string
+              description: "Human-readable error message"
+              example: "Too many requests"
+            hint:
+              type: string
+              nullable: true
+              description: "No additional data available"
+              example: null
+  headers:
+    Location:
+      description: Location of created resource
+      schema:
+        type: string
+
+  parameters:
+    database:
+      from:
+        in: query
+        description: Unix timestamp from when the data should be requested
+        name: from
+        schema:
+          type: integer
+        required: true
+        example: 1672580025
+      until:
+        in: query
+        description: Unix timestamp from when the data should be requested
+        name: until
+        schema:
+          type: integer
+        required: true
+        example: 1672666425
+      blocked:
+        in: query
+        description: Should this query return only blocked queries?
+        schema:
+          type: boolean
+        required: false
+        example: false

src/api/docs/content/specs/config.yaml

@@ -0,0 +1,835 @@
+openapi: 3.0.2
+components:
+  paths:
+    config:
+      get:
+        summary: Get current configuration of your Pi-hole
+        tags:
+          - "Pi-hole Configuration"
+        operationId: "get_config"
+        description: |
+          This API hook returns infos about the config of your Pi-hole.
+        parameters:
+          - $ref: 'config.yaml#/components/parameters/detailed'
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'config.yaml#/components/schemas/config'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  config:
+                    $ref: 'config.yaml#/components/examples/config'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+      patch:
+        summary: Change configuration of your Pi-hole
+        tags:
+          - "Pi-hole Configuration"
+        operationId: "patch_config"
+        description: |
+          This API hook allows to modify the config of your Pi-hole. This endpoint supports changing multiple properties at once when you specify several in the payload. See examples below.
+        requestBody:
+          description: Callback payload
+          content:
+            application/json:
+              schema:
+                allOf:
+                  - $ref: 'config.yaml#/components/schemas/config'
+                  - $ref: 'common.yaml#/components/schemas/took'
+              examples:
+                config_one:
+                  $ref: 'config.yaml#/components/examples/config_one'
+                config_two:
+                  $ref: 'config.yaml#/components/examples/config_two'
+                config:
+                  $ref: 'config.yaml#/components/examples/config'
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'config.yaml#/components/schemas/config'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  config:
+                    $ref: 'config.yaml#/components/examples/config'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    config_elem:
+      get:
+        summary: Get specific part of current configuration of your Pi-hole
+        tags:
+          - "Pi-hole Configuration"
+        operationId: "get_config_elem"
+        description: |
+          This API hook returns infos about the requested subset of your Pi-hole's configuration.
+          The response will be a filtered JSON object and a subset of the full `GET /config` response.
+        parameters:
+          - $ref: 'config.yaml#/components/parameters/detailed'
+          - $ref: 'config.yaml#/components/parameters/element'
+        responses:
+          '200':
+            description: OK
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    config_elem_value:
+      put:
+        summary: Add config array item
+        tags:
+          - "Pi-hole Configuration"
+        operationId: "add_array_item"
+        description: |
+          *Note:* There will be no content on success.
+        parameters:
+          - $ref: 'config.yaml#/components/parameters/element'
+          - $ref: 'config.yaml#/components/parameters/value'
+        responses:
+          '201':
+            description: Item created
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  invalid_path_depth:
+                    $ref: 'config.yaml#/components/examples/errors/bad_request/invalid_path_depth'
+                  item_already_present:
+                    $ref: 'config.yaml#/components/examples/errors/bad_request/item_already_present'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+      delete:
+        summary: Delete config array item
+        tags:
+          - "Pi-hole Configuration"
+        operationId: "delete_array_item"
+        description: |
+          *Note:* There will be no content on success.
+        parameters:
+          - $ref: 'config.yaml#/components/parameters/element'
+          - $ref: 'config.yaml#/components/parameters/value'
+        responses:
+          '204':
+            description: Item deleted
+          '404':
+            description: Item not found
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  invalid_path_depth:
+                    $ref: 'config.yaml#/components/examples/errors/bad_request/invalid_path_depth'
+                  item_already_present:
+                    $ref: 'config.yaml#/components/examples/errors/bad_request/item_already_present'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+
+  schemas:
+    config:
+      type: object
+      properties:
+        config:
+          type: object
+          properties:
+            dns:
+              type: object
+              properties:
+                upstreams:
+                  type: array
+                  items:
+                    type: string
+                CNAMEdeepInspect:
+                  type: boolean
+                blockESNI:
+                  type: boolean
+                EDNS0ECS:
+                  type: boolean
+                ignoreLocalhost:
+                  type: boolean
+                showDNSSEC:
+                  type: boolean
+                analyzeOnlyAandAAAA:
+                  type: boolean
+                piholePTR:
+                  type: string
+                replyWhenBusy:
+                  type: string
+                blockTTL:
+                  type: integer
+                hosts:
+                  type: array
+                  items:
+                    type: string
+                domainNeeded:
+                  type: boolean
+                expandHosts:
+                  type: boolean
+                domain:
+                  type: string
+                bogusPriv:
+                  type: boolean
+                dnssec:
+                  type: boolean
+                interface:
+                  type: string
+                hostRecord:
+                  type: string
+                listeningMode:
+                  type: string
+                queryLogging:
+                  type: boolean
+                cnameRecords:
+                  type: array
+                  items:
+                    type: string
+                port:
+                  type: integer
+                cache:
+                  type: object
+                  properties:
+                    size:
+                      type: integer
+                    optimizer:
+                      type: integer
+                revServers:
+                  type: array
+                  items:
+                    type: string
+                blocking:
+                  type: object
+                  properties:
+                    active:
+                      type: boolean
+                    mode:
+                      type: string
+                specialDomains:
+                  type: object
+                  properties:
+                    mozillaCanary:
+                      type: boolean
+                    iCloudPrivateRelay:
+                      type: boolean
+                reply:
+                  type: object
+                  properties:
+                    host:
+                      type: object
+                      properties:
+                        force4:
+                          type: boolean
+                        force6:
+                          type: boolean
+                        IPv4:
+                          type: string
+                          x-format: ipv4
+                        IPv6:
+                          type: string
+                          x-format: ipv6
+                    blocking:
+                      type: object
+                      properties:
+                        force4:
+                          type: boolean
+                        force6:
+                          type: boolean
+                        IPv4:
+                          type: string
+                          x-format: ipv4
+                        IPv6:
+                          type: string
+                          x-format: ipv6
+                rateLimit:
+                  type: object
+                  properties:
+                    count:
+                      type: integer
+                    interval:
+                      type: integer
+            dhcp:
+              type: object
+              properties:
+                active:
+                  type: boolean
+                start:
+                  type: string
+                  x-format: ipv4
+                end:
+                  type: string
+                  x-format: ipv4
+                router:
+                  type: string
+                  x-format: ipv4
+                netmask:
+                  type: string
+                  x-format: ipv4
+                leaseTime:
+                  type: string
+                ipv6:
+                  type: boolean
+                rapidCommit:
+                  type: boolean
+                multiDNS:
+                  type: boolean
+                logging:
+                  type: boolean
+                hosts:
+                  type: array
+                  items:
+                    type: string
+            resolver:
+              type: object
+              properties:
+                resolveIPv4:
+                  type: boolean
+                resolveIPv6:
+                  type: boolean
+                networkNames:
+                  type: boolean
+                refreshNames:
+                  type: string
+            database:
+              type: object
+              properties:
+                DBimport:
+                  type: boolean
+                maxDBdays:
+                  type: integer
+                DBinterval:
+                  type: integer
+                useWAL:
+                  type: boolean
+                network:
+                  type: object
+                  properties:
+                    parseARPcache:
+                      type: boolean
+                    expire:
+                      type: integer
+            webserver:
+              type: object
+              properties:
+                domain:
+                  type: string
+                acl:
+                  type: string
+                port:
+                  type: string
+                session:
+                  type: object
+                  properties:
+                    timeout:
+                      type: integer
+                    restore:
+                      type: boolean
+                tls:
+                  type: object
+                  properties:
+                    rev_proxy:
+                      type: boolean
+                    cert:
+                      type: string
+                paths:
+                  type: object
+                  properties:
+                    webroot:
+                      type: string
+                    webhome:
+                      type: string
+                interface:
+                  type: object
+                  properties:
+                    boxed:
+                      type: boolean
+                    theme:
+                      type: string
+                api:
+                  type: object
+                  properties:
+                    localAPIauth:
+                      type: boolean
+                    searchAPIauth:
+                      type: boolean
+                    max_sessions:
+                      type: integer
+                    prettyJSON:
+                      type: boolean
+                    password:
+                      description: |
+                        *Note:* Special write-only property used to change the password via the API.
+                      type: string
+                    pwhash:
+                      type: string
+                    totp_secret:
+                      type: string
+                    app_pwhash:
+                      type: string
+                    excludeClients:
+                      type: array
+                      items:
+                        type: string
+                    excludeDomains:
+                      type: array
+                      items:
+                        type: string
+                    maxHistory:
+                      type: integer
+                    maxClients:
+                      type: integer
+                    client_history_global_max:
+                      type: boolean
+                    allow_destructive:
+                      type: boolean
+                    temp:
+                      type: object
+                      properties:
+                        limit:
+                          type: number
+                        unit:
+                          type: string
+            files:
+              type: object
+              properties:
+                pid:
+                  type: string
+                database:
+                  type: string
+                gravity:
+                  type: string
+                gravity_tmp:
+                  type: string
+                macvendor:
+                  type: string
+                setupVars:
+                  type: string
+                pcap:
+                  type: string
+                log:
+                  type: object
+                  properties:
+                    ftl:
+                      type: string
+                    dnsmasq:
+                      type: string
+                    webserver:
+                      type: string
+            misc:
+              type: object
+              properties:
+                nice:
+                  type: integer
+                delay_startup:
+                  type: integer
+                addr2line:
+                  type: boolean
+                etc_dnsmasq_d:
+                  type: boolean
+                privacylevel:
+                  type: integer
+                dnsmasq_lines:
+                  type: array
+                  items:
+                    type: string
+                extraLogging:
+                  type: boolean
+                check:
+                  type: object
+                  properties:
+                    load:
+                      type: boolean
+                    shmem:
+                      type: integer
+                    disk:
+                      type: integer
+            debug:
+              type: object
+              properties:
+                database:
+                  type: boolean
+                networking:
+                  type: boolean
+                locks:
+                  type: boolean
+                queries:
+                  type: boolean
+                flags:
+                  type: boolean
+                shmem:
+                  type: boolean
+                gc:
+                  type: boolean
+                arp:
+                  type: boolean
+                regex:
+                  type: boolean
+                api:
+                  type: boolean
+                tls:
+                  type: boolean
+                overtime:
+                  type: boolean
+                status:
+                  type: boolean
+                caps:
+                  type: boolean
+                dnssec:
+                  type: boolean
+                vectors:
+                  type: boolean
+                resolver:
+                  type: boolean
+                edns0:
+                  type: boolean
+                clients:
+                  type: boolean
+                aliasclients:
+                  type: boolean
+                events:
+                  type: boolean
+                helper:
+                  type: boolean
+                config:
+                  type: boolean
+                inotify:
+                  type: boolean
+                webserver:
+                  type: boolean
+                extra:
+                  type: boolean
+                reserved:
+                  type: boolean
+                all:
+                  type: boolean
+    topics:
+      type: object
+      properties:
+        topics:
+          type: array
+          items:
+            type: object
+            properties:
+              name:
+                type: string
+                description: The name of the topic
+              title:
+                type: string
+                description: The tab title of the topic
+              description:
+                type: string
+                description: A human-readable description of the topic
+    server:
+      type: object
+      properties:
+        server:
+          type: array
+          items:
+            type: object
+            properties:
+              name:
+                type: string
+                description: Human-readable name of this server
+              v4:
+                type: array
+                description: Array of IPv4 addresses (if any)
+                items:
+                  type: string
+              v6:
+                type: array
+                description: Array of IPv6 addresses (if any)
+                items:
+                  type: string
+
+  examples:
+    config:
+      summary: The entire configuration
+      value:
+        config:
+          dns:
+            upstreams: [ "127.0.0.1#5353", "8.8.8.8" ]
+            CNAMEdeepInspect: true
+            blockESNI: true
+            EDNS0ECS: true
+            ignoreLocalhost: false
+            showDNSSEC: true
+            analyzeOnlyAandAAAA: false
+            piholePTR: PI.HOLE
+            replyWhenBusy: ALLOW
+            blockTTL: 2
+            hosts:
+              - "192.168.2.123 mymusicbox"
+            domainNeeded: true
+            expandHosts: true
+            domain: "lan"
+            bogusPriv: true
+            dnssec: true
+            interface: "eth0"
+            hostRecord: ""
+            listeningMode: "local"
+            queryLogging: true
+            cnameRecords:
+              - "*.example.com,default.example.com"
+              - "hourly.yetanother.com,yetanother.com,3600"
+            port: 53
+            cache:
+              size: 10000
+              optimizer: 3600
+            revServers:
+              - "true,192.168.0.0/24,192.168.0.1,lan"
+            blocking:
+              active: true
+              mode: 'NULL'
+            specialDomains:
+              mozillaCanary: true
+              iCloudPrivateRelay: true
+            reply:
+              host:
+                force4: false
+                force6: false
+                IPv4: 0.0.0.0
+                IPv6: "::"
+              blocking:
+                force4: false
+                force6: false
+                IPv4: 0.0.0.0
+                IPv6: "::"
+            rateLimit:
+              count: 0
+              interval: 0
+          dhcp:
+            active: false
+            start: "192.168.0.10"
+            end: "192.168.0.250"
+            router: "192.168.0.1"
+            netmask: "0.0.0.0"
+            leaseTime: "24h"
+            ipv6: false
+            rapidCommit: false
+            multiDNS: false
+            logging: false
+            hosts:
+              - "11:22:33:44:55:66,192.168.1.123"
+              - "11:22:33:44:55:67,192.168.1.124,hostname"
+          resolver:
+            resolveIPv4: true
+            resolveIPv6: true
+            networkNames: true
+            refreshNames: IPV4_ONLY
+          database:
+            DBimport: true
+            maxDBdays: 365
+            DBinterval: 60
+            useWAL: true
+            network:
+              parseARPcache: true
+              expire: 365
+          webserver:
+            domain: pi.hole
+            acl: "+0.0.0.0/0,::/0"
+            port: 80,[::]:80
+            session:
+              timeout: 300
+              restore: true
+            tls:
+              rev_proxy: false
+              cert: "/etc/pihole/tls.pem"
+            paths:
+              webroot: "/var/www/html"
+              webhome: "/admin/"
+            interface:
+              boxed: true
+              theme: "default-darker"
+            api:
+              localAPIauth: false
+              searchAPIauth: false
+              max_sessions: 16
+              prettyJSON: false
+              password: "********"
+              pwhash: ''
+              totp_secret: ''
+              app_pwhash: ''
+              excludeClients: [ '1\.2\.3\.4', 'localhost', 'fe80::345' ]
+              excludeDomains: [ 'google\\.de', 'pi-hole\.net' ]
+              maxHistory: 86400
+              maxClients: 10
+              client_history_global_max: true
+              allow_destructive: true
+              temp:
+                limit: 60.0
+                unit: "C"
+          files:
+            pid: "/run/pihole-FTL.pid"
+            database: "/etc/pihole/pihole-FTL.db"
+            gravity: "/etc/pihole/gravity.db"
+            gravity_tmp: "/tmp"
+            macvendor: "/etc/pihole/macvendor.db"
+            setupVars: "/etc/pihole/setupVars.conf"
+            pcap: ""
+            log:
+              ftl: "/var/log/pihole/FTL.log"
+              dnsmasq: "/var/log/pihole/pihole.log"
+              webserver: "/var/log/pihole/webserver.log"
+          misc:
+            nice: -10
+            delay_startup: 10
+            addr2line: true
+            privacylevel: 0
+            etc_dnsmasq_d: false
+            dnsmasq_lines: [ ]
+            extraLogging: false
+            check:
+              load: true
+              shmem: 90
+              disk: 90
+          debug:
+            database: false
+            networking: false
+            locks: false
+            queries: false
+            flags: false
+            shmem: false
+            gc: false
+            arp: false
+            regex: false
+            api: false
+            tls: false
+            overtime: false
+            status: false
+            caps: false
+            dnssec: false
+            vectors: false
+            resolver: false
+            edns0: false
+            clients: false
+            aliasclients: false
+            events: false
+            helper: false
+            config: false
+            inotify: false
+            webserver: false
+            extra: false
+            reserved: false
+            all: false
+    config_one:
+      summary: One option
+      value:
+        config:
+          dns:
+            CNAMEdeepInspect: true
+    config_two:
+      summary: Two options
+      value:
+        config:
+          dns:
+            specialDomains:
+              mozillaCanary: true
+          misc:
+            nice: -10
+    topics:
+      summary: All topics
+      value:
+        topics:
+          - name: dns
+            title: DNS
+            description: DNS settings
+          - name: dhcp
+            title: DHCP
+            description: DHCP settings
+    server:
+      summary: Several servers being suggested
+      value:
+        server:
+          - name: Google (ECS, DNSSEC)
+            v4: ["8.8.8.8", "8.8.4.4"]
+            v6: ["2001:4860:4860:0:0:0:0:8888","2001:4860:4860:0:0:0:0:8844"]
+          - name: OpenDNS (ECS, DNSSEC)
+            v4: ["208.67.222.222", "208.67.220.220"]
+            v6: ["2620:119:35::35","2620:119:53::53"]
+    errors:
+      bad_request:
+        invalid_path_depth:
+          summary: Invalid path
+          value:
+            error:
+              key: "bad_request"
+              message: "Invalid path depth"
+              hint: "Use, e.g., DELETE /config/dnsmasq/upstreams/127.0.0.1 to remove \"127.0.0.1\" from config.dns.upstreams"
+        item_already_present:
+          summary: Item to be added exists already
+          value:
+            error:
+              key: "bad_request"
+              message: "Item already present"
+              hint: "Uniqueness of items is enforced"
+  parameters:
+    detailed:
+      name: detailed
+      in: query
+      description: Return detailed information about the configuration
+      required: false
+      schema:
+        type: boolean
+        default: false
+        example: false
+    element:
+      in: path
+      name: element
+      schema:
+        type: string
+      required: true
+      description: config element
+      example: "dnsmasq/upstreams"
+    value:
+      in: path
+      name: value
+      schema:
+        type: string
+      required: true
+      description: config value
+      example: "8.8.8.8"

src/api/docs/content/specs/dhcp.yaml

@@ -0,0 +1,115 @@
+openapi: 3.0.2
+components:
+  paths:
+    leases:
+      get:
+        summary: Get currently active DHCP leases
+        tags:
+          - "DHCP"
+        operationId: "get_dhcp"
+        description: |
+          This API hook returns the currently active DHCP leases.
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'dhcp.yaml#/components/schemas/leases'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    leases_ip:
+      parameters:
+        - $ref: 'dhcp.yaml#/components/parameters/ip'
+      delete:
+        summary: Remove DHCP lease
+        tags:
+          - "DHCP"
+        operationId: "delete_dhcp"
+        description: |
+          This API hook removes a currently active DHCP lease.
+          Managing DHCP leases is only possible when the DHCP server is enabled.
+          *Note:* There will be no content on success.
+        responses:
+          '204':
+            description: Item deleted
+          '404':
+            description: Item not found
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  item_missing:
+                    $ref: 'dhcp.yaml#/components/examples/errors/uri_error/item_missing'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/errors/unauthorized'
+  schemas:
+    leases:
+      type: object
+      properties:
+        leases:
+          type: array
+          description: DHCP leases
+          items:
+            type: object
+            properties:
+              expires:
+                type: integer
+                description: Expiration time (0 = infinite lease, never expires)
+                example: 1675671991
+              name:
+                type: string
+                description: Hostname
+                example: "raspberrypi"
+              hwaddr:
+                type: string
+                description: Hardware (MAC) address
+                example: "00:00:00:00:00:00"
+              ip:
+                type: string
+                description: IP address
+                example: "192.168.2.111"
+              clientid:
+                type: string
+                description: Client ID
+                example: "00:00:00:00:00:00"
+  examples:
+    errors:
+      uri_error:
+        item_missing:
+          summary: DHCP lease to be deleted not specified accurately enough
+          value:
+            error:
+              key: "uri_error"
+              message: "No ip in URI"
+              hint: null
+  parameters:
+    ip:
+      in: path
+      name: ip
+      schema:
+        type: string
+      required: true
+      description: IP address of lease to be modified
+      example: 192.168.2.222

src/api/docs/content/specs/dns.yaml

@@ -0,0 +1,142 @@
+openapi: 3.0.2
+components:
+  paths:
+    blocking:
+      summary: Modify blocking status
+      get:
+        summary: Get current blocking status
+        tags:
+          - DNS control
+        operationId: "get_blocking"
+        description: |
+          The property `timer` may contain additional details concerning a temporary en-/disabling.
+          It is `null` when no timer is active (the current status is permanent).
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'dns.yaml#/components/schemas/blocking'
+                    - $ref: 'dns.yaml#/components/schemas/timer'
+                    - $ref: 'common.yaml#/components/schemas/took'
+      post:
+        summary: Change current blocking status
+        tags:
+          - DNS control
+        operationId: "set_blocking"
+        description: |
+          Change the current blocking mode by setting `blocking` to the desired value.
+          The optional `timer` object may used to set a timer. Once this timer elapsed, the opposite blocking mode is automatically set.
+          For instance, you can request `{blocking: false, timer: 60}` to disable Pi-hole for one minute.
+          Blocking will be automatically resumed afterwards.
+
+          You can terminate a possibly running timer by setting `timer` to `null` (the set mode becomes permanent).
+        requestBody:
+          description: Callback payload
+          content:
+            'application/json':
+              schema:
+                allOf:
+                  - $ref: 'dns.yaml#/components/schemas/blocking_bool'
+                  - $ref: 'dns.yaml#/components/schemas/timer'
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'dns.yaml#/components/schemas/blocking'
+                    - $ref: 'dns.yaml#/components/schemas/timer'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - oneOf:
+                      - $ref: 'dns.yaml#/components/schemas/errors/no_payload'
+                      - $ref: 'dns.yaml#/components/schemas/errors/item_missing'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+  schemas:
+    blocking:
+      type: object
+      properties:
+        blocking:
+          type: string
+          description: Blocking status
+          enum:
+            - "enabled"
+            - "disabled"
+            - "failed"
+            - "unknown"
+          example: "enabled"
+    blocking_bool:
+      type: object
+      properties:
+        blocking:
+          type: boolean
+          description: Blocking status
+          default: true
+          example: true
+    timer:
+      type: object
+      properties:
+        timer:
+          type: number
+          description: Remaining seconds until blocking mode is automatically changed
+          nullable: true
+          example: 15.0
+
+    errors:
+      item_missing:
+        type: object
+        description: Item to be modified is missing
+        properties:
+          error:
+            type: object
+            properties:
+              key:
+                type: string
+                description: "Machine-readable error type"
+                example: "body_error"
+              message:
+                type: string
+                description: "Human-readable error message"
+                example: "No \\\"blocking\\\" boolean in body data"
+              hint:
+                type: string
+                nullable: true
+                description: "Additional data (if available)"
+                example: null
+      no_payload:
+        type: object
+        description: No JSON payload found
+        properties:
+          error:
+            type: object
+            properties:
+              key:
+                type: string
+                description: "Machine-readable error type"
+                example: "bad_request"
+              message:
+                type: string
+                description: "Human-readable error message"
+                example: "Invalid request body data (no valid JSON)"
+              hint:
+                type: string
+                nullable: true
+                description: "Additional data (if available)"
+                example: null

src/api/docs/content/specs/docs.yaml

@@ -0,0 +1,29 @@
+openapi: 3.0.2
+components:
+  paths:
+    docs:
+      summary: Pi-hole's API documentation
+      get:
+        summary: Get the embedded API documentation rendered as HTML
+        tags:
+          - "Documentation"
+        operationId: "get_docs"
+        security: []
+        description: |
+          This API hook returns the embedded API documentation rendered as HTML.
+        responses:
+          '200':
+            description: OK
+            content:
+              text/html:
+                schema:
+                  description: HTML document
+                  type: string
+                  example: |
+                    <!DOCTYPE html>
+                    <html lang="en">
+                    <head>
+                      <meta charset="utf-8">
+                      <title>API Documentation</title>
+                    </body>
+                    </html>

src/api/docs/content/specs/domains.yaml

@@ -0,0 +1,545 @@
+openapi: 3.0.2
+components:
+  paths:
+    type_kind_domain:
+      summary: Modify domains and regular expressions
+      parameters:
+        - $ref: 'domains.yaml#/components/parameters/type'
+        - $ref: 'domains.yaml#/components/parameters/kind'
+        - $ref: 'domains.yaml#/components/parameters/domain'
+      get:
+        summary: Get domain
+        tags:
+          - "Domain management"
+        operationId: "get_domains"
+        description: |
+          `{type}`, `{kind}`, and `{domain}` are optional. Specifying any of these may result in only a subset of the available data being returned.
+
+          Valid combinations are:
+          - `/api/domains` (all domains)
+          - `/api/domains/abc.com` (all domains identical to `abc.com`)
+          - `/api/domains/allow` (only allowed domains)
+          - `/api/domains/allow/abc.com` (only allowed domains identical to `abc.com`)
+          - `/api/domains/allow/exact` (only exactly allowed domains)
+          - `/api/domains/allow/exact/abc.com` (only exactly allowed domain identical to `abc.com`)
+          - `/api/domains/allow/regex` (only allowed regex domains)
+          - `/api/domains/allow/regex/abc.com` (only allowed regex domains identical to `abc.com`)
+          - `/api/domains/deny` (only denied domains)
+          - `/api/domains/deny/abc.com` (only denied domains identical to `abc.com`)
+          - `/api/domains/deny/exact` (only exactly denied domains)
+          - `/api/domains/deny/exact/abc.com` (only exactly denied domain identical to `abc.com`)
+          - `/api/domains/deny/regex` (only denied regex domains)
+          - `/api/domains/deny/regex/abc.com` (only denied regex domains identical to `abc.com`)
+          - `/api/domains/exact` (allowed and denied exact domains)
+          - `/api/domains/exact/abc.com` (allowed and denied exact domains identical to `abc.com`)
+          - `/api/domains/regex` (allowed and denied regex domains)
+          - `/api/domains/regex/abc.com` (allowed and denied regex domains identical to `abc.com`)
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'domains.yaml#/components/schemas/domains/get'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  domains:
+                    $ref: 'domains.yaml#/components/examples/domains'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+      put:
+        summary: Replace domain
+        tags:
+          - "Domain management"
+        operationId: "replace_domain"
+        description: |
+          Items may be updated by replacing them. `{type}`, `{kind}`, and `{domain}` are required.
+
+          Ensure to send all the required parameters (such as `comment`) to ensure these properties are retained.
+          The read-only fields `id` and `date_added` are preserved, `date_modified` is automatically updated on success.
+
+          You can move existing domains to another list type/kind by `PUT`ting the domain to the new destination by specifying the optional fields `type` and `kind`.
+          Example:
+          Use `PUT allow/exact/abc.com` with `type="deny", kind="exact"` to change `abc.com` from exact denied to exact allowed. Make sure to always specify *both* values.
+
+          When adding/replacing a regular expression, ensure that `{domain}` is properly URI-escaped.
+        requestBody:
+          description: Callback payload
+          content:
+            'application/json':
+              schema:
+                allOf:
+                  - $ref: 'domains.yaml#/components/schemas/domains/put'
+                  - $ref: 'common.yaml#/components/schemas/took'
+        responses:
+          '201':
+            description: Created domain
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'domains.yaml#/components/schemas/domains/get' # identical to GET
+                    - $ref: 'domains.yaml#/components/schemas/lists_processed'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  domains:
+                    $ref: 'domains.yaml#/components/examples/domains'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  list_imprecise:
+                    $ref: 'domains.yaml#/components/examples/errors/uri_error/list_imprecise'
+                  item_missing:
+                    $ref: 'domains.yaml#/components/examples/errors/uri_error/item_missing'
+                  no_payload:
+                    $ref: 'domains.yaml#/components/examples/errors/bad_request/no_payload'
+                  duplicate:
+                    $ref: 'domains.yaml#/components/examples/errors/database_error/duplicate'
+                  invalid_regex:
+                    $ref: 'domains.yaml#/components/examples/errors/regex_error/invalid_regex'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+      delete:
+        summary: Delete domain
+        tags:
+          - "Domain management"
+        operationId: "delete_domain"
+        description: |
+          *Note:* There will be no content on success.
+        responses:
+          '204':
+            description: Item deleted
+          '404':
+            description: Item not found
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  list_imprecise:
+                    $ref: 'domains.yaml#/components/examples/errors/uri_error/list_imprecise'
+                  item_missing:
+                    $ref: 'domains.yaml#/components/examples/errors/uri_error/item_missing'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    type_kind:
+      parameters:
+        - $ref: 'domains.yaml#/components/parameters/type'
+        - $ref: 'domains.yaml#/components/parameters/kind'
+      post:
+        summary: Add new domain
+        tags:
+          - "Domain management"
+        operationId: "add_domain"
+        description: |
+          Creates a new domain in the `domains` object. This may be either an exact domain or a regex, depending on `{kind}`.
+          Both `{type}` and `{kind}` are mandatory for this endpoint.
+          The `{domain}` itself is specified in the request body (POST JSON).
+
+          On success, a new resource is created at `/domains/{type}/{kind}/{domain}`.
+
+          The `database_error` with message `UNIQUE constraint failed` error indicates that the same entry (`domain`, `type`, `kind`) already exists.
+
+          When adding a regular expression, ensure the request body is properly JSON-escaped.
+        requestBody:
+          description: Callback payload
+          content:
+            application/json:
+              schema:
+                $ref: 'domains.yaml#/components/schemas/domains/post'
+        responses:
+          '201':
+            description: Created domain
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'domains.yaml#/components/schemas/domains/get' # identical to GET
+                    - $ref: 'domains.yaml#/components/schemas/lists_processed'
+                    - $ref: 'common.yaml#/components/schemas/took'
+            headers:
+              Location:
+                $ref: 'common.yaml#/components/headers/Location'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  list_imprecise:
+                    $ref: 'domains.yaml#/components/examples/errors/uri_error/list_imprecise'
+                  no_payload:
+                    $ref: 'domains.yaml#/components/examples/errors/bad_request/no_payload'
+                  duplicate:
+                    $ref: 'domains.yaml#/components/examples/errors/database_error/duplicate'
+                  invalid_regex:
+                    $ref: 'domains.yaml#/components/examples/errors/regex_error/invalid_regex'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    batchDelete:
+      summary: Delete multiple domains
+      post:
+        summary: Delete multiple domains
+        tags:
+          - "Domain management"
+        operationId: "batchDelete_domains"
+        description: |
+          *Note:* There will be no content on success.
+        requestBody:
+          description: Callback payload
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    item:
+                      type: string
+                      description: Domain to delete
+                      example: "example.com"
+                    type:
+                      type: string
+                      description: Type of domain to delete
+                      enum:
+                        - "allow"
+                        - "deny"
+                      example: "allow"
+                    kind:
+                      type: string
+                      description: Kind of domain to delete
+                      enum:
+                        - "exact"
+                        - "regex"
+                      example: "exact"
+        responses:
+          '204':
+            description: Items deleted
+          '404':
+            description: Item not found
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  no_payload:
+                    $ref: 'domains.yaml#/components/examples/errors/bad_request/no_payload'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+  schemas:
+    domains:
+      get:
+        type: object
+        properties:
+          domains:
+            type: array
+            description: Array of domains
+            items:
+              allOf:
+                - $ref: 'domains.yaml#/components/schemas/domain_object'
+                - $ref: 'domains.yaml#/components/schemas/unicode'
+                - $ref: 'domains.yaml#/components/schemas/type'
+                - $ref: 'domains.yaml#/components/schemas/kind'
+                - $ref: 'domains.yaml#/components/schemas/comment'
+                - $ref: 'domains.yaml#/components/schemas/groups'
+                - $ref: 'domains.yaml#/components/schemas/enabled'
+                - $ref: 'domains.yaml#/components/schemas/readonly'
+      put:
+        allOf:
+          - $ref: 'domains.yaml#/components/schemas/type'
+          - $ref: 'domains.yaml#/components/schemas/kind'
+          - $ref: 'domains.yaml#/components/schemas/comment'
+          - $ref: 'domains.yaml#/components/schemas/groups'
+          - $ref: 'domains.yaml#/components/schemas/enabled'
+      post:
+        allOf:
+          - $ref: 'domains.yaml#/components/schemas/domain_maybe_array'
+          - $ref: 'domains.yaml#/components/schemas/comment'
+          - $ref: 'domains.yaml#/components/schemas/groups'
+          - $ref: 'domains.yaml#/components/schemas/enabled'
+    domain:
+      description: Domain
+      type: string
+      example: testdomain.com
+    unicode:
+      type: object
+      properties:
+        unicode:
+          description: Unicode domain (may be different from `domain` if punycode-encoding is used)
+          type: string
+          example: "äbc.com"
+    domain_array:
+      description: array of domains
+      type: array
+      items:
+        type: string
+      example: ["testdomain.com", "otherdomain.de"]
+    domain_maybe_array:
+      type: object
+      properties:
+        domain:
+          oneOf:
+            - $ref: 'domains.yaml#/components/schemas/domain'
+            - $ref: 'domains.yaml#/components/schemas/domain_array'
+    domain_object:
+      type: object
+      properties:
+        domain:
+          $ref: 'domains.yaml#/components/schemas/domain'
+    type:
+      type: object
+      properties:
+        type:
+          description: String specifying domain type
+          type: string
+          enum:
+            - "allow"
+            - "deny"
+          example: "allow"
+    kind:
+      type: object
+      properties:
+        kind:
+          description: String specifying domain kind
+          type: string
+          enum:
+            - "exact"
+            - "regex"
+          example: "exact"
+    comment:
+      type: object
+      properties:
+        comment:
+          description: User-provided free-text comment for this domain
+          type: string
+          nullable: true
+          default: null
+          example: Some comment describing this domain
+    groups:
+      type: object
+      properties:
+        groups:
+          description: Array of group IDs
+          type: array
+          items:
+            type: integer
+    enabled:
+      type: object
+      properties:
+        enabled:
+          description: Status of domain
+          type: boolean
+          default: true
+    readonly:
+      type: object
+      properties:
+        id:
+          description: Database ID
+          type: integer
+          readOnly: true
+        date_added:
+          description: Unix timestamp of domain addition
+          type: integer
+          readOnly: true
+        date_modified:
+          description: Unix timestamp of last domain modification
+          type: integer
+          readOnly: true
+    lists_processed:
+      type: object
+      properties:
+        processed:
+          type: object
+          nullable: true
+          description: |
+            Object containing the number of domains that were successfully
+            added to the database and the number of domains that could not be
+            added to the database.
+          properties:
+            success:
+              description: |
+                Array of domains that were successfully added to the database.
+              type: array
+              items:
+                type: object
+                properties:
+                  item:
+                    description: Domain that was added to the database
+                    type: string
+            errors:
+              description: |
+                Array of errors that occurred during processing.
+              type: array
+              items:
+                type: object
+                properties:
+                  item:
+                    description: Domain that could not be added to the database
+                    type: string
+                  error:
+                    description: Error message
+                    type: string
+          example:
+            success:
+              - item: "example.com"
+              - item: "example3.com"
+            errors:
+              - item: "example2.com"
+                error: "UNIQUE constraint failed: domainlist.domain"
+  examples:
+    domains:
+      summary: Example domains
+      value:
+        domains:
+          - domain: "allowed.com"
+            unicode: "allowed.com"
+            type: allow
+            kind: exact
+            comment: null
+            groups:
+              - 0
+            enabled: true
+            id: 299
+            date_added: 1611239095
+            date_modified: 1612163756
+          - domain: "xn--4ca.com"
+            unicode: "ä.com"
+            type: allow
+            kind: regex
+            comment: "Some text"
+            groups:
+              - 0
+            enabled: true
+            id: 305
+            date_added: 1611240635
+            date_modified: 1611241276
+        took: 0.012
+        processed: null
+    errors:
+      uri_error:
+        list_imprecise:
+          summary: List not specified precisely enough
+          value:
+            error:
+              key: "uri_error"
+              message: "Invalid request: Specify list to modify more precisely"
+              hint: null
+        item_missing:
+          summary: Domain to be modified is missing
+          value:
+            error:
+              key: "uri_error"
+              message: "Invalid request: Specify item in URI"
+              hint: null
+      bad_request:
+        no_payload:
+          summary: No JSON payload found
+          value:
+            error:
+              key: "bad_request"
+              message: "Invalid request body data (no valid JSON)"
+              hint: null
+      database_error:
+        duplicate:
+          summary: Database error
+          value:
+            error:
+              key: "database_error"
+              message: "Could not add to gravity database"
+              hint: "UNIQUE constraint failed: domainlist.domain, domainlist.type"
+      regex_error:
+        invalid_regex:
+          summary: Invalid regex rejected
+          value:
+            error:
+              key: "regex_error"
+              message: "Regex validation failed"
+              hint: "Missing ']'"
+
+  parameters:
+    type:
+      in: path
+      name: type
+      description: Type (allowed or denied domain)
+      schema:
+        type: string
+        enum:
+          - allow
+          - deny
+      required: true
+      example: allow
+    kind:
+      in: path
+      name: kind
+      description: Kind (exact domain or regular expression)
+      schema:
+        type: string
+        enum:
+          - exact
+          - regex
+      required: true
+      example: exact
+    domain:
+      in: path
+      name: domain
+      description: Domain
+      schema:
+        type: string
+      required: true
+      example: testdomain.com

src/api/docs/content/specs/endpoints.yaml

@@ -0,0 +1,154 @@
+openapi: 3.0.2
+components:
+  paths:
+    endpoints:
+      get:
+        summary: Get list of available API endpoints
+        tags:
+          - "FTL information"
+        operationId: "get_endpoints"
+        description: |
+          This API hook returns the list of all available API endpoints.
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'endpoints.yaml#/components/schemas/endpoints'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+  schemas:
+    endpoints:
+      type: object
+      properties:
+        endpoints:
+          type: object
+          description: Endpoints
+          properties:
+            get:
+              type: array
+              items:
+                type: object
+                properties:
+                  uri:
+                    type: string
+                    description: URI
+                  parameters:
+                    type: string
+                    description: Parameters
+              example:
+                - uri: "/api/dns/blocking"
+                  parameters: ''
+                - uri: "/api/dns/cache"
+                  parameters: ''
+                - uri: "/api/dns/port"
+                  parameters: ''
+                - uri: "/api/domains"
+                  parameters: ''
+                - uri: "/api/groups"
+                  parameters: ''
+                - uri: "/api/lists"
+                  parameters: ''
+            post:
+              type: array
+              items:
+                type: object
+                properties:
+                  uri:
+                    type: string
+                    description: URI
+                  parameters:
+                    type: string
+                    description: Parameters
+              example:
+                - uri: "/api/dns/blocking"
+                  parameters: ''
+                - uri: "/api/dns/cache"
+                  parameters: ''
+                - uri: "/api/dns/port"
+                  parameters: ''
+                - uri: "/api/domains"
+                  parameters: ''
+                - uri: "/api/groups"
+                  parameters: ''
+                - uri: "/api/lists"
+                  parameters: ''
+            put:
+              type: array
+              items:
+                type: object
+                properties:
+                  uri:
+                    type: string
+                    description: URI
+                  parameters:
+                    type: string
+                    description: Parameters
+              example:
+                - uri: "/api/dns/blocking"
+                  parameters: ''
+                - uri: "/api/dns/cache"
+                  parameters: ''
+                - uri: "/api/dns/port"
+                  parameters: ''
+                - uri: "/api/domains"
+                  parameters: ''
+                - uri: "/api/groups"
+                  parameters: ''
+                - uri: "/api/lists"
+                  parameters: ''
+            patch:
+              type: array
+              items:
+                properties:
+                  uri:
+                    type: string
+                    description: URI
+                  parameters:
+                    type: string
+                    description: Parameters
+              example:
+                - uri: "/api/dns/blocking"
+                  parameters: ''
+                - uri: "/api/dns/cache"
+                  parameters: ''
+                - uri: "/api/dns/port"
+                  parameters: ''
+                - uri: "/api/domains"
+                  parameters: ''
+                - uri: "/api/groups"
+                  parameters: ''
+                - uri: "/api/lists"
+                  parameters: ''
+            delete:
+              type: array
+              items:
+                properties:
+                  uri:
+                    type: string
+                    description: URI
+                  parameters:
+                    type: string
+                    description: Parameters
+              example:
+                - uri: "/api/dns/blocking"
+                  parameters: ''
+                - uri: "/api/dns/cache"
+                  parameters: ''
+                - uri: "/api/dns/port"
+                  parameters: ''
+                - uri: "/api/domains"
+                  parameters: ''
+                - uri: "/api/groups"
+                  parameters: ''
+                - uri: "/api/lists"
+                  parameters: ''

src/api/docs/content/specs/groups.yaml

@@ -0,0 +1,408 @@
+openapi: 3.0.2
+components:
+  paths:
+    name:
+      summary: Modify group
+      parameters:
+        - $ref: 'groups.yaml#/components/parameters/name'
+      get:
+        summary: Get groups
+        tags:
+          - "Group management"
+        operationId: "get_groups"
+        description: |
+          `{name}` is optional. Specifying it will result in only the requested group being returned.
+
+          Valid combinations are:
+          - `/api/groups` (all groups)
+          - `/api/groups/my_group` (group identical to `my_group`)
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'groups.yaml#/components/schemas/groups/get'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  groups:
+                    $ref: 'groups.yaml#/components/examples/groups'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+      put:
+        summary: Replace group
+        tags:
+          - "Group management"
+        operationId: "replace_group"
+        description: |
+          Items may be updated by replacing them. `{name}` is required.
+
+          Ensure to send all the required parameters (such as `comment`) to ensure these properties are retained.
+          By specifying a different `name`, the group with the former name as specified in the URI will be renamed.
+          The read-only fields `id` and `date_added` are preserved, `date_modified` is automatically updated on success.
+        requestBody:
+          description: Callback payload
+          content:
+            application/json:
+              schema:
+                $ref: 'groups.yaml#/components/schemas/groups/put'
+        responses:
+          '201':
+            description: Created item
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'groups.yaml#/components/schemas/groups/get' # identical to GET
+                    - $ref: 'groups.yaml#/components/schemas/lists_processed'
+                    - $ref: 'common.yaml#/components/schemas/took'
+            headers:
+              Location:
+                $ref: 'common.yaml#/components/headers/Location'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  item_missing:
+                    $ref: 'groups.yaml#/components/examples/errors/uri_error/item_missing'
+                  no_payload:
+                    $ref: 'groups.yaml#/components/examples/errors/bad_request/no_payload'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+      delete:
+        summary: Delete group
+        tags:
+          - "Group management"
+        operationId: "delete_group"
+        description: |
+          *Note:* There will be no content on success.
+        responses:
+          '204':
+            description: Item deleted
+          '404':
+            description: Item not found
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  item_missing:
+                    $ref: 'groups.yaml#/components/examples/errors/uri_error/item_missing'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    direct:
+      post:
+        summary: Add new group
+        tags:
+          - "Group management"
+        operationId: "add_group"
+        description: |
+          Creates a new group in the `groups` object. The `{group}` itself is specified in the request body (POST JSON).
+
+          On success, a new resource is created at `/groups/{name}`.
+
+          The `database_error` with message `UNIQUE constraint failed` error indicates that a group with the same name already exists.
+        requestBody:
+          description: Callback payload
+          content:
+            application/json:
+                schema:
+                  $ref: 'groups.yaml#/components/schemas/groups/post'
+        responses:
+          '201':
+            description: Created item
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'groups.yaml#/components/schemas/groups/get' # identical to GET
+                    - $ref: 'groups.yaml#/components/schemas/lists_processed'
+                    - $ref: 'common.yaml#/components/schemas/took'
+            headers:
+              Location:
+                $ref: 'common.yaml#/components/headers/Location'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  no_payload:
+                    $ref: 'groups.yaml#/components/examples/errors/bad_request/no_payload'
+                  duplicate:
+                    $ref: 'groups.yaml#/components/examples/errors/database_error/duplicate'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    batchDelete:
+      post:
+        summary: Delete multiple groups
+        tags:
+          - "Group management"
+        operationId: "batchDelete_groups"
+        description: |
+          Deletes multiple groups in the `groups` object. The `{groups}` themselves are specified in the request body (POST JSON).
+
+          On success, a new resource is created at `/groups/{name}`.
+
+          The `database_error` with message `UNIQUE constraint failed` error indicates that a group with the same name already exists.
+        requestBody:
+          description: Callback payload
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    item:
+                      type: string
+                      description: group name
+                example:
+                  - "item": "test1"
+                  - "item": "test2"
+        responses:
+          '204':
+            description: Items deleted
+          '404':
+            description: Item not found
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  no_payload:
+                    $ref: 'groups.yaml#/components/examples/errors/bad_request/no_payload'
+                  duplicate:
+                    $ref: 'groups.yaml#/components/examples/errors/database_error/duplicate'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+  schemas:
+    groups:
+      get:
+        type: object
+        properties:
+          groups:
+            type: array
+            items:
+              allOf:
+                - $ref: 'groups.yaml#/components/schemas/name_object'
+                - $ref: 'groups.yaml#/components/schemas/comment'
+                - $ref: 'groups.yaml#/components/schemas/enabled'
+                - $ref: 'groups.yaml#/components/schemas/readonly'
+      put:
+        allOf:
+          # Can rename group
+          - $ref: 'groups.yaml#/components/schemas/name_object'
+          - $ref: 'groups.yaml#/components/schemas/comment'
+          - $ref: 'groups.yaml#/components/schemas/enabled'
+      post:
+        allOf:
+          - $ref: 'groups.yaml#/components/schemas/name_maybe_array'
+          - $ref: 'groups.yaml#/components/schemas/comment'
+          - $ref: 'groups.yaml#/components/schemas/enabled'
+    name:
+      description: Group name
+      type: string
+      example: test_group
+    name_array:
+      description: array of group names
+      type: array
+      items:
+        type: string
+      example: ["test1", "test2", "test3"]
+    name_maybe_array:
+      type: object
+      properties:
+        name:
+          oneOf:
+            - $ref: 'groups.yaml#/components/schemas/name'
+            - $ref: 'groups.yaml#/components/schemas/name_array'
+    name_object:
+      type: object
+      properties:
+        name:
+          $ref: 'groups.yaml#/components/schemas/name'
+    comment:
+      type: object
+      properties:
+        comment:
+          description: User-provided free-text comment for this group
+          type: string
+          nullable: true
+          default: null
+          example: Some comment for this group
+    enabled:
+      type: object
+      properties:
+        enabled:
+          description: Status of item
+          type: boolean
+          default: true
+          example: true
+    readonly:
+      type: object
+      properties:
+        id:
+          description: Database ID
+          type: integer
+          readOnly: true
+          example: 1
+        date_added:
+          description: Unix timestamp of item addition
+          type: integer
+          readOnly: true
+          example: 1611239095
+        date_modified:
+          description: Unix timestamp of last item modification
+          type: integer
+          readOnly: true
+          example: 1611239099
+    lists_processed:
+      type: object
+      properties:
+        processed:
+          type: object
+          nullable: true
+          description: |
+            Object containing the number of groups that were successfully
+            added to the database and the number of groups that could not be
+            added to the database.
+          properties:
+            success:
+              description: |
+                Array of groups that were successfully added to the database.
+              type: array
+              items:
+                type: object
+                properties:
+                  item:
+                    description: Group that was added to the database
+                    type: string
+            errors:
+              description: |
+                Array of errors that occurred during processing.
+              type: array
+              items:
+                type: object
+                properties:
+                  item:
+                    description: Group that could not be added to the database
+                    type: string
+                  error:
+                    description: Error message
+                    type: string
+          example:
+            success:
+              - item: "Home-Automation"
+              - item: "Children"
+            errors:
+              - item: "Garden"
+                error: "UNIQUE constraint failed: group.name"
+  examples:
+    groups:
+      value:
+        groups:
+          - name: "Default"
+            comment: "The default group"
+            enabled: true
+            id: 0
+            date_added: 1594670974
+            date_modified: 1611157897
+          - name: "a"
+            comment: null
+            enabled: true
+            id: 5
+            date_added: 1604871899
+            date_modified: 1604871899
+        took: 0.003
+        processed: null
+    errors:
+      uri_error:
+        item_missing:
+          summary: Group to be modified is missing
+          value:
+            error:
+              key: "uri_error"
+              message: "Invalid request: Specify item in URI"
+              hint: null
+      bad_request:
+        no_payload:
+          summary: No JSON payload found
+          value:
+            error:
+              key: "bad_request"
+              message: "Invalid request body data (no valid JSON)"
+              hint: null
+      database_error:
+        duplicate:
+          summary: Database error
+          value:
+            error:
+              key: "database_error"
+              message: "Could not add to gravity database"
+              hint: "UNIQUE constraint failed: grouplist.group"
+
+  parameters:
+    name:
+      in: path
+      name: name
+      schema:
+        type: string
+      required: true
+      description: Group name
+      example: test_group

src/api/docs/content/specs/history.yaml

@@ -0,0 +1,226 @@
+openapi: 3.0.2
+components:
+  paths:
+    history:
+      get:
+        summary: Get activity graph data
+        tags:
+          - Metrics
+        operationId: "get_activity_metrics"
+        description: |
+          Request data needed to generate the \"Query over last 24 hours\" graph. The sum of the values in the individual data arrays may be smaller than the total number of queries for the corresponding timestamp. The remaining queries are queries that do not fit into the shown categories (e.g. database busy, unknown status queries, etc.).
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'history.yaml#/components/schemas/total_history'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    database_history:
+      get:
+        summary: Get activity graph data (long-term data)
+        tags:
+          - Metrics
+        operationId: "get_activity_metrics_database"
+        description: |
+          Request long-term data needed to generate the activity graph
+        parameters:
+          - $ref: 'common.yaml#/components/parameters/database/from'
+          - $ref: 'common.yaml#/components/parameters/database/until'
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'history.yaml#/components/schemas/total_history'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+
+    clients:
+      get:
+        summary: Get per-client activity graph data
+        tags:
+          - Metrics
+        operationId: "get_client_metrics"
+        description: |
+          Request data needed to generate the \"Client activity over last 24 hours\" graph.
+          This endpoint returns the top N clients, sorted by total number of queries within 24 hours. If N is set to 0, all clients will be returned.
+          The client name is only available if the client's IP address can be resolved to a hostname.
+
+          The last client returned is a special client that contains the total number of queries that were not sent by any of the other shown clients , i.e. queries that were sent by clients that are not in the top N. This client is always present, even if it has 0 queries and can be identified by the special name "other clients" (mind the space in the hostname) and the IP address "0.0.0.0".
+
+          Note that, due to privacy settings, the returned data may also be empty.
+        parameters:
+          - $ref: 'history.yaml#/components/parameters/clients/N'
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'history.yaml#/components/schemas/client_history'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    database_clients:
+      get:
+        summary: Get per-client activity graph data (long-term data)
+        tags:
+          - Metrics
+        operationId: "get_client_metrics_database"
+        description: |
+          Request long-term data needed to generate the client activity graph
+        parameters:
+          - $ref: 'common.yaml#/components/parameters/database/from'
+          - $ref: 'common.yaml#/components/parameters/database/until'
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'history.yaml#/components/schemas/client_history'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+
+  schemas:
+    total_history:
+      type: object
+      properties:
+        history:
+          type: array
+          description: Data array
+          items:
+            type: object
+            properties:
+              timestamp:
+                type: number
+                description: Timestamp
+              total:
+                type: integer
+                description: Total number of queries
+              cached:
+                type: integer
+                description: Number of cached queries
+              blocked:
+                type: integer
+                description: Number of blocked queries
+              forwarded:
+                type: integer
+                description: Number of forwarded queries
+          example:
+            - timestamp: 1511819900.539157
+              total: 2134
+              cached: 525
+              blocked: 413
+              forwarded: 1196
+            - timestamp: 1511820500.583821
+              total: 2014
+              cached: 52
+              blocked: 43
+              forwarded: 1910
+    client_history:
+      type: object
+      properties:
+        clients:
+          type: object
+          description: Data corresponding to the individual clients
+          additionalProperties:
+            type: object
+            properties:
+              name:
+                type: string
+                nullable: true
+                description: Client name
+              total:
+                type: integer
+                description: Total number of queries
+          example:
+            "127.0.0.1":
+              name: localhost
+              total: 13428
+            "::1":
+              name: ip6-localnet
+              total: 2100
+            "192.168.1.1":
+              name: null
+              total: 254
+            "::":
+              name: "pi.hole"
+              total: 29
+            "0.0.0.0":
+              name: "other clients"
+              total: 14
+        history:
+          type: array
+          description: Data array
+          items:
+            type: object
+            properties:
+              timestamp:
+                type: number
+                description: Timestamp
+              data:
+                type: object
+                description: Data corresponding to the individual clients
+                additionalProperties:
+                  type: integer
+                  description: Number of queries for the corresponding client
+          example:
+            - timestamp: 1511819900.539157
+              data:
+                "127.0.0.1": 35
+                "::1": 63
+                "192.168.1.1": 20
+                "::": 9
+                "0.0.0.0": 0
+            - timestamp: 1511820500.583821
+              data:
+                "127.0.0.1": 10
+                "::1": 44
+                "192.168.1.1": 56
+                "::": 52
+  parameters:
+    clients:
+      N:
+        in: query
+        description: Maximum number of clients to return, setting this to 0 will return all clients
+        name: N
+        schema:
+          type: integer
+        required: false
+        example: 20

src/api/docs/content/specs/lists.yaml

@@ -0,0 +1,463 @@
+openapi: 3.0.2
+components:
+  paths:
+    list:
+      summary: Modify list
+      parameters:
+        - $ref: 'lists.yaml#/components/parameters/list'
+        - $ref: 'lists.yaml#/components/parameters/listtype'
+      get:
+        summary: Get lists
+        tags:
+          - "List management"
+        operationId: "get_lists"
+        description: |
+          `{list}` is optional. Specifying it will result in only the requested list being returned.
+
+          Valid combinations are:
+          - `/api/lists` (all lists)
+          - `/api/lists/my_list` (list identical to `my_list`)
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'lists.yaml#/components/schemas/lists/get'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  lists:
+                    $ref: 'lists.yaml#/components/examples/lists'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+      put:
+        summary: Replace list
+        tags:
+          - "List management"
+        operationId: "replace_lists"
+        description: |
+          Items may be updated by replacing them. `{list}` is required.
+
+          Ensure to send all the required parameters (such as `comment`) to ensure these properties are retained.
+          The read-only fields `id` and `date_added` are preserved, `date_modified` is automatically updated on success.
+        requestBody:
+          description: Callback payload
+          content:
+            application/json:
+              schema:
+                $ref: 'lists.yaml#/components/schemas/lists/put'
+        responses:
+          '201':
+            description: Created item
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'lists.yaml#/components/schemas/lists/get' # identical to GET
+                    - $ref: 'lists.yaml#/components/schemas/lists_processed'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  item_missing:
+                    $ref: 'lists.yaml#/components/examples/errors/uri_error/item_missing'
+                  no_payload:
+                    $ref: 'lists.yaml#/components/examples/errors/bad_request/no_payload'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+      delete:
+        summary: Delete list
+        tags:
+          - "List management"
+        operationId: "delete_lists"
+        description: |
+          *Note:* There will be no content on success.
+        responses:
+          '204':
+            description: Item deleted
+          '404':
+            description: Item not found
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  item_missing:
+                    $ref: 'lists.yaml#/components/examples/errors/uri_error/item_missing'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    direct:
+      post:
+        summary: Add new list
+        tags:
+          - "List management"
+        operationId: "add_list"
+        description: |
+          Creates a new list in the `lists` object. The `{list}` itself is specified in the request body (POST JSON).
+
+          lists may be described either by their IP addresses (IPv4 and IPv6 are supported),
+          IP subnets (CIDR notation, like `192.168.2.0/24`), their MAC addresses (like `12:34:56:78:9A:BC`), by their hostnames (like `localhost`), or by the interface they are connected to (prefaced with a colon, like `:eth0`).</p>
+
+          Note that list recognition by IP addresses (incl. subnet ranges) is preferred over MAC address, host name or interface recognition as the two latter will only be available after some time.
+          Furthermore, MAC address recognition only works for devices at most one networking hop away from your Pi-hole.
+
+          On success, a new resource is created at `/lists/{list}`.
+
+          The `database_error` with message `UNIQUE constraint failed` error indicates that this list already exists.
+        requestBody:
+          description: Callback payload
+          content:
+            application/json:
+              schema:
+                $ref: 'lists.yaml#/components/schemas/lists/post'
+        responses:
+          '201':
+            description: Created item
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'lists.yaml#/components/schemas/lists/get'
+                    - $ref: 'lists.yaml#/components/schemas/lists_processed'
+                    - $ref: 'common.yaml#/components/schemas/took'
+            headers:
+              Location:
+                $ref: 'common.yaml#/components/headers/Location'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  no_payload:
+                    $ref: 'lists.yaml#/components/examples/errors/bad_request/no_payload'
+                  duplicate:
+                    $ref: 'lists.yaml#/components/examples/errors/database_error/duplicate'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    batchDelete:
+      post:
+        summary: Delete lists
+        tags:
+          - "List management"
+        operationId: "batchDelete_lists"
+        description: |
+          Deletes multiple lists in the `lists` object. The `{list}`s themselves are specified in the request body (POST JSON).
+
+          On success, a new resource is created at `/lists/{list}`.
+
+          The `database_error` with message `UNIQUE constraint failed` error indicates that this list already exists.
+        requestBody:
+          description: Callback payload
+          content:
+            application/json:
+              schema:
+                $ref: 'lists.yaml#/components/schemas/lists/post'
+        responses:
+          '204':
+            description: Items deleted
+          '404':
+            description: Item not found
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  no_payload:
+                    $ref: 'lists.yaml#/components/examples/errors/bad_request/no_payload'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+  schemas:
+    lists:
+      get:
+        type: object
+        properties:
+          lists:
+            type: array
+            description: Array of lists
+            items:
+              allOf:
+                - $ref: 'lists.yaml#/components/schemas/address_object'
+                - $ref: 'lists.yaml#/components/schemas/type'
+                - $ref: 'lists.yaml#/components/schemas/comment'
+                - $ref: 'lists.yaml#/components/schemas/groups'
+                - $ref: 'lists.yaml#/components/schemas/enabled'
+                - $ref: 'lists.yaml#/components/schemas/readonly'
+      put:
+        allOf:
+          - $ref: 'lists.yaml#/components/schemas/comment'
+          - $ref: 'lists.yaml#/components/schemas/type'
+          - $ref: 'lists.yaml#/components/schemas/groups'
+          - $ref: 'lists.yaml#/components/schemas/enabled'
+      post:
+        allOf:
+          - $ref: 'lists.yaml#/components/schemas/address_maybe_array'
+          - $ref: 'lists.yaml#/components/schemas/type'
+          - $ref: 'lists.yaml#/components/schemas/comment'
+          - $ref: 'lists.yaml#/components/schemas/groups'
+          - $ref: 'lists.yaml#/components/schemas/enabled'
+    address:
+      description: Address of the list
+      type: string
+      example: https://hosts-file.net/ad_servers.txt
+    address_array:
+      description: array of list addresses
+      type: array
+      items:
+        type: string
+      example: ["https://hosts-file.net/ad_servers.txt"]
+    address_maybe_array:
+      type: object
+      properties:
+        address:
+          oneOf:
+            - $ref: 'lists.yaml#/components/schemas/address'
+            - $ref: 'lists.yaml#/components/schemas/address_array'
+    address_object:
+      type: object
+      properties:
+        address:
+          $ref: 'lists.yaml#/components/schemas/address'
+    type:
+      type: object
+      properties:
+        type:
+          description: Type of list
+          type: string
+          enum:
+            - "allow"
+            - "block"
+          example: "block"
+    comment:
+      type: object
+      properties:
+        comment:
+          description: User-provided free-text comment for this list
+          type: string
+          nullable: true
+          default: null
+          example: Some comment for this list
+    groups:
+      type: object
+      properties:
+        groups:
+          description: Array of group IDs
+          type: array
+          default: [0]
+          items:
+            type: integer
+    enabled:
+      type: object
+      properties:
+        enabled:
+          description: Status of domain
+          type: boolean
+          default: true
+          example: true
+    readonly:
+      type: object
+      properties:
+        id:
+          description: Database ID
+          type: integer
+          readOnly: true
+          example: 1
+        date_added:
+          description: Unix timestamp of item addition
+          type: integer
+          readOnly: true
+          example: 1611239095
+        date_modified:
+          description: Unix timestamp of last item modification
+          type: integer
+          readOnly: true
+          example: 1611239099
+        date_updated:
+          description: Unix timestamp of last update of list content
+          type: integer
+          readOnly: true
+          example: 1608435667
+        number:
+          description: Number of VALID domains on this list
+          type: integer
+          readOnly: true
+          example: 20566
+        invalid_domains:
+          description: Number of INVALID domains on this list
+          type: integer
+          readOnly: true
+          example: 0
+        abp_entries:
+          description: Number of ABP entries on this list
+          type: integer
+          readOnly: true
+          example: 0
+        status:
+          description: List status
+          type: integer
+          readOnly: true
+          example: 2
+    lists_processed:
+      type: object
+      properties:
+        processed:
+          type: object
+          nullable: true
+          description: |
+            Object containing the number of lists that were successfully
+            added to the database and the number of lists that could not be
+            added to the database.
+          properties:
+            success:
+              description: |
+                Array of lists that were successfully added to the database.
+              type: array
+              items:
+                type: object
+                properties:
+                  item:
+                    description: List that was added to the database
+                    type: string
+            errors:
+              description: |
+                Array of errors that occurred during processing.
+              type: array
+              items:
+                type: object
+                properties:
+                  item:
+                    description: List that could not be added to the database
+                    type: string
+                  error:
+                    description: Error message
+                    type: string
+          example:
+            success:
+              - item: "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts"
+              - item: "https://mirror1.malwaredomains.com/files/justdomains"
+            errors:
+              - item: "https://raw.githubusercontent.com/abc/def/master/hosts"
+                error: "UNIQUE constraint failed: adlist.address"
+  examples:
+    lists:
+      value:
+        lists:
+          - address: "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts"
+            comment: "Migrated from /etc/pihole/adlists.list"
+            groups:
+              - 0
+            enabled: true
+            id: 1
+            date_added: 1594670974
+            date_modified: 1595279300
+          - address: "https://mirror1.malwaredomains.com/files/justdomains"
+            comment: "Migrated from /etc/pihole/adlists.list"
+            groups:
+              - 0
+            enabled: true
+            id: 2
+            date_added: 1594670974
+            date_modified: 1594670974
+        took: 0.012
+        processed: null
+    errors:
+      uri_error:
+        item_missing:
+          summary: List to be modified is missing
+          value:
+            error:
+              key: "uri_error"
+              message: "Invalid request: Specify item in URI"
+              hint: null
+      bad_request:
+        no_payload:
+          summary: No JSON payload found
+          value:
+            error:
+              key: "bad_request"
+              message: "Invalid request body data (no valid JSON)"
+              hint: null
+      database_error:
+        duplicate:
+          summary: Database error
+          value:
+            error:
+              key: "database_error"
+              message: "Could not add to gravity database"
+              hint: "UNIQUE constraint failed: listlist.list"
+  parameters:
+    list:
+      in: path
+      name: list
+      schema:
+        type: string
+      required: true
+      description: Address of the list
+      example: https://hosts-file.net/ad_servers.txt
+    listtype:
+      in: query
+      name: type
+      schema:
+        type: string
+        enum:
+          - "allow"
+          - "block"
+      required: false
+      description: Type of list, optional
+      example: block

src/api/docs/content/specs/logs.yaml

@@ -0,0 +1,158 @@
+openapi: 3.0.2
+components:
+  paths:
+    logs:
+      dnsmasq:
+        get:
+          summary: Get DNS log content
+          tags:
+            - "FTL information"
+          operationId: "get_dns_log"
+          description: |
+            This API hook returns content from the log of the embedded DNS resolver `dnsmasq`.
+
+            Every successful request will return a `nextID`.
+            This ID can be used on the next request to only get lines which were added *after* the last request.
+            This makes periodic polling for new log lines easy as no check for duplicated log lines is necessary.
+            The expected behavior for an immediate re-request of a log line with the same ID is an empty response.
+            As soon as the next message arrived, this will be included in your request and `nextID` is incremented by one.
+          parameters:
+            - $ref: 'logs.yaml#/components/parameters/logs/dnsmasq/nextID'
+          responses:
+            '200':
+              description: OK
+              content:
+                application/json:
+                  schema:
+                    allOf:
+                      - $ref: 'logs.yaml#/components/schemas/logs/dnsmasq/log'
+                      - $ref: 'common.yaml#/components/schemas/took'
+            '401':
+              description: Unauthorized
+              content:
+                application/json:
+                  schema:
+                    allOf:
+                      - $ref: 'common.yaml#/components/errors/unauthorized'
+                      - $ref: 'common.yaml#/components/schemas/took'
+      ftl:
+        get:
+          summary: Get DNS log content
+          tags:
+            - "FTL information"
+          operationId: "get_ftl_log"
+          description: |
+            This API hook returns content from the log of FTL.
+
+            Every successful request will return a `nextID`.
+            This ID can be used on the next request to only get lines which were added *after* the last request.
+            This makes periodic polling for new log lines easy as no check for duplicated log lines is necessary.
+            The expected behavior for an immediate re-request of a log line with the same ID is an empty response.
+            As soon as the next message arrived, this will be included in your request and `nextID` is incremented by one.
+          parameters:
+            - $ref: 'logs.yaml#/components/parameters/logs/dnsmasq/nextID'
+          responses:
+            '200':
+              description: OK
+              content:
+                application/json:
+                  schema:
+                    allOf:
+                      - $ref: 'logs.yaml#/components/schemas/logs/dnsmasq/log'
+                      - $ref: 'common.yaml#/components/schemas/took'
+            '401':
+              description: Unauthorized
+              content:
+                application/json:
+                  schema:
+                    allOf:
+                      - $ref: 'common.yaml#/components/errors/unauthorized'
+                      - $ref: 'common.yaml#/components/schemas/took'
+      webserver:
+        get:
+          summary: Get DNS log content
+          tags:
+            - "FTL information"
+          operationId: "get_webserver_log"
+          description: |
+            This API hook returns content from the log of the embedded CivetWeb HTTP server.
+
+            Every successful request will return a `nextID`.
+            This ID can be used on the next request to only get lines which were added *after* the last request.
+            This makes periodic polling for new log lines easy as no check for duplicated log lines is necessary.
+            The expected behavior for an immediate re-request of a log line with the same ID is an empty response.
+            As soon as the next message arrived, this will be included in your request and `nextID` is incremented by one.
+          parameters:
+            - $ref: 'logs.yaml#/components/parameters/logs/dnsmasq/nextID'
+          responses:
+            '200':
+              description: OK
+              content:
+                application/json:
+                  schema:
+                    allOf:
+                      - $ref: 'logs.yaml#/components/schemas/logs/dnsmasq/log'
+                      - $ref: 'common.yaml#/components/schemas/took'
+            '401':
+              description: Unauthorized
+              content:
+                application/json:
+                  schema:
+                    allOf:
+                      - $ref: 'common.yaml#/components/errors/unauthorized'
+                      - $ref: 'common.yaml#/components/schemas/took'
+  schemas:
+    logs:
+      dnsmasq:
+        log:
+          type: object
+          properties:
+            log:
+              type: array
+              description: Request headers
+              items:
+                type: object
+                properties:
+                  timestamp:
+                    type: number
+                    description: Unix timestamp of log line creation (server time)
+                  message:
+                    type: string
+                    description: Log line content
+                  prio:
+                    type: string
+                    nullable: true
+                    description: Log line priority (if available)
+              example:
+                - timestamp: 1611729969.0
+                  message: "started, version pi-hole-2.84 cachesize 10000"
+                - timestamp: 1611729969.0
+                  message: "reading /etc/resolv.conf"
+                - timestamp: 1611729969.0
+                  message: "read /etc/hosts - 7 addresses"
+                - timestamp: 1611729969.0
+                  message: "query[A] connectivity-check.ubuntu.com from 127.0.0.1"
+            nextID:
+              type: integer
+              description: Next ID to query if checking for new log lines
+              example: 229
+            pid:
+              type: integer
+              description: Process ID of FTL. When this changes, FTL was restarted and nextID should be reset to 0.
+              example: 2258
+            file:
+              type: string
+              description: Path to respective log file on disk
+              example: /var/log/pihole/pihole.log
+
+  parameters:
+    logs:
+      dnsmasq:
+        nextID:
+          in: query
+          description: (Optional) ID of next line to return
+          name: nextID
+          schema:
+            type: integer
+          required: false
+          example: 219

src/api/docs/content/specs/main.yaml

@@ -0,0 +1,301 @@
+openapi: 3.0.2
+info:
+  title: Pi-hole API
+  version: "6.0"
+  contact:
+    name: Pi-hole API Support
+    url: https://discourse.pi-hole.net
+  license:
+    name: European Union Public Licence (EUPL)
+    url: https://joinup.ec.europa.eu/community/eupl/og_page/eupl
+  description: >
+    The Pi-hole API is organized around [REST](http://en.wikipedia.org/wiki/Representational_State_Transfer).
+    Our API has predictable resource-oriented URLs, accepts and returns reliable UTF-8 [JavaScript Object Notation (JSON)-encoded](http://www.json.org/) data for all API responses, and uses standard HTTP response codes and verbs.
+
+    Most (but not all) endpoints require authentication.
+    API endpoints requiring authentication will fail with code `401 Unauthorized` when used outside a valid session.
+servers:
+  - url: 'https://{url}:{port}/{path}'
+    variables:
+      url:
+        description: URL or address of your Pi-hole
+        default: pi.hole
+      port:
+        description: Port of your Pi-hole's API (HTTPS)
+        default: "443"
+      path:
+        description: Path where your Pi-hole's API is hosted
+        default: api
+  - url: 'http://{url}:{port}/{path}'
+    variables:
+      url:
+        description: URL or address of your Pi-hole
+        default: pi.hole
+      port:
+        description: Port of your Pi-hole's API
+        default: "80"
+      path:
+        description: Path where your Pi-hole's API is hosted
+        default: api
+
+tags:
+  - name: "Authentication"
+    description: Methods used to authentaticate with the API
+  - name: "Metrics"
+    description: Methods used to get usage data from your Pi-hole
+  - name: "DNS control"
+    description: Methods used to control the behavior of your Pi-hole
+  - name: "DHCP server control"
+    description: Methods used to control the DHCP server of your Pi-hole
+  - name: "Group management"
+    description: Methods used to manage groups on your Pi-hole
+  - name: "Domain management"
+    description: Methods used to manage domains on your Pi-hole
+  - name: "Client management"
+    description: Methods used to manage clients on your Pi-hole
+  - name: "List management"
+    description: Methods used to manage lists on your Pi-hole
+  - name: "FTL information"
+    description: Methods used to gather advanced data from your Pi-hole
+  - name: "Pi-hole configuration"
+    description: Methods used to configure your Pi-hole
+  - name: "Network information"
+    description: Methods used to gather advanced information about your network
+  - name: "Actions"
+    description: Methods used to trigger certain actions on your Pi-hole
+
+
+
+paths:
+  /auth:
+    $ref: 'auth.yaml#/components/paths/auth'
+
+  /auth/session/{id}:
+    $ref: 'auth.yaml#/components/paths/session'
+
+  /auth/sessions:
+    $ref: 'auth.yaml#/components/paths/session_list'
+
+  /auth/totp:
+    $ref: 'auth.yaml#/components/paths/totp'
+
+  /auth/app:
+    $ref: 'auth.yaml#/components/paths/app'
+
+  /stats/summary:
+    $ref: 'stats.yaml#/components/paths/summary'
+
+  /stats/database/summary:
+    $ref: 'stats.yaml#/components/paths/database_summary'
+
+  /stats/upstreams:
+    $ref: 'stats.yaml#/components/paths/upstreams'
+
+  /stats/database/upstreams:
+    $ref: 'stats.yaml#/components/paths/database_upstreams'
+
+  /stats/top_domains:
+    $ref: 'stats.yaml#/components/paths/top_domains'
+
+  /stats/database/top_domains:
+    $ref: 'stats.yaml#/components/paths/database_top_domains'
+
+  /stats/top_clients:
+    $ref: 'stats.yaml#/components/paths/top_clients'
+
+  /stats/database/top_clients:
+    $ref: 'stats.yaml#/components/paths/database_top_clients'
+
+  /stats/query_types:
+    $ref: 'stats.yaml#/components/paths/query_types'
+
+  /stats/database/query_types:
+    $ref: 'stats.yaml#/components/paths/database_query_types'
+
+  /stats/recent_blocked:
+    $ref: 'stats.yaml#/components/paths/recent_blocked'
+
+  /history:
+    $ref: 'history.yaml#/components/paths/history'
+
+  /history/clients:
+    $ref: 'history.yaml#/components/paths/clients'
+
+  /history/database:
+    $ref: 'history.yaml#/components/paths/database_history'
+
+  /history/database/clients:
+    $ref: 'history.yaml#/components/paths/database_clients'
+
+  /queries:
+    $ref: 'queries.yaml#/components/paths/queries'
+
+  /queries/suggestions:
+    $ref: 'queries.yaml#/components/paths/suggestions'
+
+  /dns/blocking:
+    $ref: 'dns.yaml#/components/paths/blocking'
+
+  /domains/{type}/{kind}/{domain}:
+    $ref: 'domains.yaml#/components/paths/type_kind_domain'
+
+  /domains/{type}/{kind}:
+    $ref: 'domains.yaml#/components/paths/type_kind'
+
+  /domains:batchDelete:
+    $ref: 'domains.yaml#/components/paths/batchDelete'
+
+  /groups/{name}:
+    $ref: 'groups.yaml#/components/paths/name'
+
+  /groups:
+    $ref: 'groups.yaml#/components/paths/direct'
+
+  /groups:batchDelete:
+    $ref: 'groups.yaml#/components/paths/batchDelete'
+
+  /clients/{client}:
+    $ref: 'clients.yaml#/components/paths/client'
+
+  /clients:
+    $ref: 'clients.yaml#/components/paths/direct'
+
+  /clients:batchDelete:
+    $ref: 'clients.yaml#/components/paths/batchDelete'
+
+  /clients/_suggestions:
+    $ref: 'clients.yaml#/components/paths/suggestions'
+
+  /lists/{list}:
+    $ref: 'lists.yaml#/components/paths/list'
+
+  /lists:
+    $ref: 'lists.yaml#/components/paths/direct'
+
+  /lists:batchDelete:
+    $ref: 'lists.yaml#/components/paths/batchDelete'
+
+  /info/client:
+    $ref: 'info.yaml#/components/paths/client'
+
+  /info/system:
+    $ref: 'info.yaml#/components/paths/system'
+
+  /info/database:
+    $ref: 'info.yaml#/components/paths/database'
+
+  /info/ftl:
+    $ref: 'info.yaml#/components/paths/ftl'
+
+  /info/host:
+    $ref: 'info.yaml#/components/paths/host'
+
+  /info/sensors:
+    $ref: 'info.yaml#/components/paths/sensors'
+
+  /info/version:
+    $ref: 'info.yaml#/components/paths/version'
+
+  /info/messages:
+    $ref: 'info.yaml#/components/paths/messages'
+
+  /info/messages/{message_id}:
+    $ref: 'info.yaml#/components/paths/messages_with_id'
+
+  /info/messages/count:
+    $ref: 'info.yaml#/components/paths/messages_count'
+
+  /info/metrics:
+    $ref: 'info.yaml#/components/paths/metrics'
+
+  /info/login:
+    $ref: 'info.yaml#/components/paths/login'
+
+  /logs/dnsmasq:
+    $ref: 'logs.yaml#/components/paths/logs/dnsmasq'
+
+  /logs/ftl:
+    $ref: 'logs.yaml#/components/paths/logs/ftl'
+
+  /logs/webserver:
+    $ref: 'logs.yaml#/components/paths/logs/webserver'
+
+  /endpoints:
+    $ref: 'endpoints.yaml#/components/paths/endpoints'
+
+  /config:
+    $ref: 'config.yaml#/components/paths/config'
+
+  /config/{element}:
+    $ref: 'config.yaml#/components/paths/config_elem'
+
+  /config/{element}/{value}:
+    $ref: 'config.yaml#/components/paths/config_elem_value'
+
+  /network/devices:
+    $ref: 'network.yaml#/components/paths/devices'
+
+  /network/devices/{device_id}:
+    $ref: 'network.yaml#/components/paths/devices_id'
+
+  /network/gateway:
+    $ref: 'network.yaml#/components/paths/gateway'
+
+  /network/interfaces:
+    $ref: 'network.yaml#/components/paths/interfaces'
+
+  /teleporter:
+    $ref: 'teleporter.yaml#/components/paths/teleporter'
+
+  /action/gravity:
+    $ref: 'action.yaml#/components/paths/gravity'
+
+  /action/restartdns:
+    $ref: 'action.yaml#/components/paths/restartdns'
+
+  /action/flush/logs:
+    $ref: 'action.yaml#/components/paths/flush_logs'
+
+  /action/flush/arp:
+    $ref: 'action.yaml#/components/paths/flush_arp'
+
+  /dhcp/leases:
+    $ref: 'dhcp.yaml#/components/paths/leases'
+
+  /dhcp/leases/{ip}:
+    $ref: 'dhcp.yaml#/components/paths/leases_ip'
+
+  /search/{domain}:
+    $ref: 'search.yaml#/components/paths/search'
+
+  /docs:
+    $ref: 'docs.yaml#/components/paths/docs'
+
+components:
+  securitySchemes:
+    query_sid:
+      type: apiKey
+      in: query
+      name: sid
+    cookie_sid:
+      type: apiKey
+      in: cookie
+      name: sid
+    header_sid:
+      type: apiKey
+      in: header
+      name: sid
+    x_header_sid:
+      type: apiKey
+      in: header
+      name: X-FTL-SID
+    query_password:
+      type: apiKey
+      in: query
+      name: password
+
+security:
+  - query_sid: []
+  - cookie_sid: []
+  - header_sid: []
+  - x_header_sid: []

src/api/docs/content/specs/network.yaml

@@ -0,0 +1,306 @@
+
+openapi: 3.0.2
+components:
+  paths:
+    gateway:
+      get:
+        summary: Get info about the gateway of your Pi-hole
+        tags:
+          - "Network information"
+        operationId: "get_gateway"
+        description: |
+          This API hook returns infos about the gateway of your Pi-hole.
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'network.yaml#/components/schemas/gateway'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    interfaces:
+      get:
+        summary: Get info about the interfaces of your Pi-hole
+        tags:
+          - "Network information"
+        operationId: "get_interfaces"
+        description: |
+          This API hook returns infos about the networking interfaces of your Pi-hole.
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'network.yaml#/components/schemas/interfaces'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    devices:
+      get:
+        summary: Get info about the devices in your local network as seen by your Pi-hole
+        tags:
+          - "Network information"
+        operationId: "get_network"
+        description: |
+          This API hook returns infos about the devices in your local network as seen by your Pi-hole. By default, this number of shown devices is limited to 10. Devices are ordered by when your Pi-hole has received the last query from this device (most recent first)
+        parameters:
+          - $ref: 'network.yaml#/components/parameters/devices/max_devices'
+          - $ref: 'network.yaml#/components/parameters/devices/max_addresses'
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'network.yaml#/components/schemas/devices'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    devices_id:
+      delete:
+        summary: Delete a device from the network table
+        tags:
+          - "Network information"
+        operationId: "delete_device"
+        description: |
+          This API hook deletes a device from the network table. This will also remove all associated IP addresses and hostnames.
+        parameters:
+          - $ref: 'network.yaml#/components/parameters/devices/device_id'
+        responses:
+          '204':
+            description: No Content (deleted)
+          '404':
+            description: Not found
+            content:
+              application/json:
+                schema:
+                  $ref: 'common.yaml#/components/schemas/took'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/bad_request'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+  schemas:
+    gateway:
+      type: object
+      properties:
+        address:
+          type: string
+          description: Address of the gateway
+          example: "192.168.0.1"
+        interface:
+          type: string
+          description: Interface of your Pi-hole connected to the gateway
+          example: "eth0"
+    interfaces:
+      type: object
+      properties:
+        interfaces:
+          type: array
+          description: Interface information
+          items:
+            type: object
+            properties:
+              name:
+                type: string
+                nullable: true
+                description: Interface name
+              default:
+                type: boolean
+                description: If the interface is the default gateway
+              carrier:
+                type: boolean
+                description: If the interface is connected
+              speed:
+                type: integer
+                description: Speed of the interface in Mbit/s (-1 if not applicable)
+              tx:
+                type: object
+                properties:
+                  num:
+                    type: number
+                    description: Number of transmitted data since boot
+                  unit:
+                    type: string
+                    description: Unit of transmitted data since boot
+              rx:
+                type: object
+                properties:
+                  num:
+                    type: number
+                    description: Number of received data since boot
+                  unit:
+                    type: string
+                    description: Unit of received data since boot
+              ipv4:
+                type: array
+                nullable: true
+                description: Array of associated IPv4 addresses
+                items:
+                  type: string
+              ipv6:
+                type: array
+                nullable: true
+                description: Array of associated IPv6 addresses
+                items:
+                  type: string
+          example:
+            - name: "eth0"
+              default: true
+              carrier: true
+              speed: 1000
+              tx:
+                num: 10.4
+                unit: "MB"
+              rx:
+                num: 8.1
+                unit: "MB"
+              ipv4: ["192.168.0.123"]
+              ipv6: ["fe80::1234:5678:9abc:def0", "2001:db8::1234:5678:9abc:def0"]
+            - name: "wlan0"
+              default: false
+              carrier: false
+              speed: -1
+              tx:
+                num: 0
+                unit: "B"
+              rx:
+                num: 0
+                unit: "B"
+              ipv4: []
+              ipv6: []
+            - name: "wg0"
+              default: false
+              carrier: true
+              speed: -1
+              tx:
+                num: 170.3
+                unit: "kB"
+              rx:
+                num: 222.3
+                unit: "kB"
+              ipv4: ["10.1.0.1"]
+              ipv6: ["fd00:4711::1"]
+    devices:
+      type: object
+      properties:
+        devices:
+          type: array
+          description: Array of devices
+          items:
+            type: object
+            properties:
+              id:
+                type: integer
+                description: Device network table ID
+                example: 1
+              hwaddr:
+                type: string
+                description: MAC address of this device
+                example: 00:11:22:33:44:55
+              interface:
+                type: string
+                description: Interface this device is connected to
+                example: enp2s0
+              firstSeen:
+                type: integer
+                description: Unix timestamp when this device was first seen by your Pi-hole
+                example: 1664623620
+              lastQuery:
+                type: integer
+                description: Unix timestamp when your Pi-hole received the last query from this device
+                example: 1664688620
+              numQueries:
+                type: integer
+                description: Total number of queries your Pi-hole has received from this device
+                example: 585462
+              macVendor:
+                type: string
+                nullable: true
+                description: Vendor name associated with the device's MAC address (if available)
+                example: "Digital Data Communications Asia Co.,Ltd"
+              ips:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    ip:
+                      type: string
+                      description: Associated IP address (can be IPv4 or IPv6)
+                      example: "192.168.1.51"
+                    name:
+                      type: string
+                      description: Associated hostname (can be null)
+                      nullable: true
+                      example: ubuntu-server
+                    lastSeen:
+                      type: integer
+                      description: Unix timestamp when your Pi-hole has seen this address the last time
+                      example: 1664688620
+                    nameUpdated:
+                      type: integer
+                      description: Unix timestamp when device updated its hostname the last time
+                      example: 1664688620
+
+
+  parameters:
+    devices:
+      max_devices:
+        in: query
+        description: (Optional) Maximum number of devices to show
+        name: max_devices
+        schema:
+          type: integer
+        required: false
+        example: 10
+      max_addresses:
+        in: query
+        description: (Optional) Maximum number of addresses to show per device
+        name: max_addresses
+        schema:
+          type: integer
+        required: false
+        example: 3
+      device_id:
+        in: path
+        description: Device ID
+        name: device_id
+        schema:
+          type: integer
+        required: true
+        example: 1

src/api/docs/content/specs/queries.yaml

@@ -0,0 +1,320 @@
+openapi: 3.0.2
+components:
+  paths:
+    queries:
+      get:
+        summary: Get queries
+        tags:
+          - Metrics
+        operationId: "get_queries"
+        description: |
+          Request query details.
+          Query parameters may be used to limit the number of results.
+
+          By default, this API callback returns the most recent 100 queries.
+          This can be changed using the parameter `n`.
+
+          This callback allows for fine-grained filtering by various parameters.
+          All query parameters are all optional and can be combined in any way:
+
+          - Only show queries *from* a given timestamp on: Use parameter `from`
+          - Only show queries *until* a given timestamp: Use parameter `until`
+          - Only show queries sent to a specific upstream destination (may also be `cache` or `blocklist`): Use parameter `upstream`
+          - Only show queries for specific domains: Use parameter `domain`
+          - Only show queries for specific clients: Use parameter `client`
+
+          By default, the returned queries always start at the most recent query.
+          This can be changed by supplying the parameter `cursor`.
+          Each result of this API callback contains a `cursor` pointing the beginning of the next `n` queries chunk.
+          This provides a very fast and lightweight server-side pagination implementation.
+
+          If wildcards are supported for a parameter, you may specify `*` at any position in the parameter to match any number of characters.
+        parameters:
+          - in: query
+            name: from
+            description: Get queries from...
+            required: false
+            schema:
+              type: number
+          - in: query
+            name: until
+            description: Get queries until...
+            required: false
+            schema:
+              type: number
+          - in: query
+            name: length
+            description: Number of results to return
+            required: false
+            schema:
+              type: integer
+          - in: query
+            name: start
+            description: Offset from first record
+            required: false
+            schema:
+              type: integer
+          - in: query
+            name: cursor
+            description: |
+              Database ID of the most recent query to be shown
+            required: false
+            schema:
+              type: integer
+          - in: query
+            name: domain
+            description: Filter by specific domain (wildcards supported)
+            required: false
+            schema:
+              type: string
+          - in: query
+            name: client_ip
+            description: Filter by specific client IP address (wildcards supported)
+            required: false
+            schema:
+              type: string
+          - in: query
+            name: client_name
+            description: Filter by specific client hostname (wildcards supported)
+            required: false
+            schema:
+              type: string
+          - in: query
+            name: upstream
+            description: Filter by specific upstream (wildcards supported)
+            required: false
+            schema:
+              type: string
+          - in: query
+            name: type
+            description: Filter by specific query type (A, AAAA, ...)
+            required: false
+            schema:
+              type: string
+          - in: query
+            name: status
+            description: Filter by specific query status (GRAVITY, FORWARDED, ...)
+            required: false
+            schema:
+              type: string
+          - in: query
+            name: reply
+            description: Filter by specific reply type (NODATA, NXDOMAIN, ...)
+            required: false
+            schema:
+              type: string
+          - in: query
+            name: dnssec
+            description: Filter by specific DNSSEC status (SECURE, INSECURE, ...)
+            required: false
+            schema:
+              type: string
+          - in: query
+            name: disk
+            description: Load queries from on-disk database rather than from in-memory
+            required: false
+            schema:
+              enum:
+                - true
+                - false
+              default: false
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'queries.yaml#/components/schemas/queries'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+
+    suggestions:
+      get:
+        summary: Get query filter suggestions
+        tags:
+          - Metrics
+        operationId: "get_suggestions"
+        description: |
+          This endpoint provides suggestions for filters suitable to be used with /queries
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'queries.yaml#/components/schemas/suggestions'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+
+  schemas:
+    queries:
+      type: object
+      properties:
+        queries:
+          type: array
+          description: Data array
+          items:
+            type: object
+            properties:
+              id:
+                type: integer
+                description: Query ID in the long-term database
+              time:
+                type: number
+                description: Timestamp
+              type:
+                type: string
+                description: Query type
+              domain:
+                type: string
+                description: Queried domain
+              cname:
+                type: string
+                description: Domain blocked during deep CNAME inspection
+                nullable: true
+              status:
+                type: string
+                description: Query status
+                nullable: true
+              client:
+                type: object
+                properties:
+                  ip:
+                    type: string
+                    description: Requesting client's IP address
+                  name:
+                    type: string
+                    description: Requesting client's hostname (if available)
+                    nullable: true
+              dnssec:
+                type: string
+                description: DNSSEC status
+                nullable: true
+              reply:
+                type: object
+                properties:
+                  type:
+                    type: string
+                    description: Reply type
+                    nullable: true
+                  time:
+                    type: number
+                    description: Time until the response was received (ms, negative if N/A)
+              list_id:
+                type: integer
+                description: ID of corresponding database table (adlist for anti-/gravity, else domainlist) (`NULL` if N/A)
+                nullable: true
+              upstream:
+                type: string
+                description: IP or name + port of upstream server
+                nullable: true
+          example:
+            - time: 1581907991.539157
+              type: "A"
+              domain: "community.stoplight.io"
+              cname: null
+              status: "FORWARDED"
+              client:
+                ip: "192.168.0.14"
+                name: "desktop.lan"
+              dnssec: "INSECURE"
+              reply:
+                type: "IP"
+                time: 19
+              list_id: NULL
+              upstream: "localhost#5353"
+              dbid: 112421354
+            - time: 1581907871.583821
+              type: "AAAA"
+              domain: "api.github.com"
+              cname: null
+              status: "FORWARDED"
+              client:
+                ip: "127.0.0.1"
+                name: "localhost"
+              dnssec: "UNKNOWN"
+              reply:
+                type: "IP"
+                time: 12.3
+              list_id: NULL
+              upstream: "localhost#5353"
+              dbid: 112421355
+        cursor:
+          type: integer
+          description: Database ID of most recent query to show
+          example: 175881
+        recordsTotal:
+          type: integer
+          description: Total number of available queries
+          example: 1234
+        recordsFiltered:
+          type: integer
+          description: Number of available queries after filtering
+          example: 1234
+        draw:
+          type: integer
+          description: DataTables-specific integer (echos input value)
+          example: 1
+    suggestions:
+      type: object
+      properties:
+        suggestions:
+          type: object
+          properties:
+            domain:
+              type: array
+              description: Array of suggested domains
+              items:
+                type: string
+              example: ["pi-hole.net","amazon.com"]
+            client_ip:
+              type: array
+              description: Array of suggested client IP addresses
+              items:
+                type: string
+            client_name:
+              type: array
+              description: Array of suggested client names
+              items:
+                type: string
+            upstream:
+              type: array
+              description: Array of suggested upstreams
+              items:
+                type: string
+            type:
+              type: array
+              description: Array of suggested query types
+              items:
+                type: string
+            status:
+              type: array
+              description: Array of suggested query statuses
+              items:
+                type: string
+            reply:
+              type: array
+              description: Array of suggested query replies
+              items:
+                type: string
+            dnssec:
+              type: array
+              description: Array of suggested DNSSEC statuses
+              items:
+                type: string

src/api/docs/content/specs/search.yaml

@@ -0,0 +1,246 @@
+openapi: 3.0.2
+components:
+  paths:
+    search:
+      parameters:
+        - $ref: 'search.yaml#/components/parameters/domain'
+      get:
+        summary: Search domains in Pi-hole's lists
+        tags:
+          - "List management"
+        operationId: "get_search"
+        parameters:
+          - $ref: 'search.yaml#/components/parameters/partial'
+          - $ref: 'search.yaml#/components/parameters/N'
+          - $ref: 'search.yaml#/components/parameters/debug'
+        description: |
+          Search for domains in Pi-hole's list. The specified domain is automatically converted to lowercase.
+          The optional parameters `N` and `partial` limit the maximum number of returned records and whether partial matches should be returned, respectively.
+          There is a hard upper limit of `N` defined in FTL (currently set to 10,000) to ensure that the response is not too large.
+          ABP matches are not returned when partial matching is requested.
+          Depending on the value of the config option webserver.api.searchAPIauth, local clients may not need to authenticate for this endpoint.
+          International domains names (IDNs) are internally converted to punycode before matching.
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'search.yaml#/components/schemas/search'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+  schemas:
+    search:
+      type: object
+      properties:
+        search:
+          type: object
+          properties:
+            domains:
+              type: array
+              items:
+                type: object
+                properties:
+                  domain:
+                    type: string
+                    example: "blockeddomain.com"
+                  comment:
+                    type: string
+                    description: Optional comment
+                    nullable: true
+                    example: "I needed to block this because of XYZ"
+                  enabled:
+                    type: boolean
+                    description: Whether this entry is enabled
+                    example: true
+                  type:
+                    description: String specifying domain type
+                    type: string
+                    enum:
+                      - "allow"
+                      - "deny"
+                    example: "allow"
+                  kind:
+                    description: String specifying domain kind
+                    type: string
+                    enum:
+                      - "exact"
+                      - "regex"
+                    example: "exact"
+                  id:
+                    type: integer
+                    description: Database ID
+                    example: 7
+                  date_added:
+                    type: integer
+                    description: Unix timestamp of addition
+                    example: 1664624500
+                  date_modified:
+                    type: integer
+                    description: Unix timestamp of last modification
+                    example: 1664624500
+                  groups:
+                    type: array
+                    description: Array of IDs corresponding to associated groups
+                    items:
+                      type: integer
+                    example: [0,1,2]
+            gravity:
+              type: array
+              items:
+                type: object
+                properties:
+                  domain:
+                    type: string
+                    example: "doubleclick.net"
+                  address:
+                    type: string
+                    description: Address of the list this domain was found on
+                    example: "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts"
+                  comment:
+                    type: string
+                    description: Optional comment of the list
+                    nullable: true
+                    example: "I needed to block this because of XYZ"
+                  enabled:
+                    type: boolean
+                    description: Whether this list is enabled
+                    example: true
+                  id:
+                    type: integer
+                    description: Database ID of the associated list
+                    example: 0
+                  type:
+                    type: string
+                    description: String specifying list type
+                    enum:
+                      - "allow"
+                      - "block"
+                  date_added:
+                    type: integer
+                    description: Unix timestamp of list addition
+                    example: 1664624500
+                  date_modified:
+                    type: integer
+                    description: Unix timestamp of last list modification
+                    example: 1664624500
+                  date_updated:
+                    type: integer
+                    description: Unix timestamp of last local update of this list
+                    example: 1664624500
+                  number:
+                    type: integer
+                    description: Number of entries in the list
+                    example: 7
+                  invalid_domains:
+                    type: integer
+                    description: Number of invalid domains in the list
+                    example: 0
+                  abp_entries:
+                    type: integer
+                    description: Number of ABP entries in the list
+                    example: 0
+                  status:
+                    type: integer
+                    description: Status of the list
+                    example: 1
+                  groups:
+                    type: array
+                    description: Array of IDs corresponding to associated groups
+                    items:
+                      type: integer
+                    example: [0,1,2]
+            parameters:
+              type: object
+              properties:
+                partial:
+                  type: boolean
+                  description: Whether partial matching was requested
+                  example: false
+                N:
+                  type: integer
+                  description: Maximum number of results to be returned (per type)
+                  example: 20
+                domain:
+                  type: string
+                  description: (Part of) domain to be searched for
+                  example: "doubleclick.net"
+                debug:
+                  type: boolean
+                  description: Whether debug information was requested
+                  example: false
+            results:
+              type: object
+              properties:
+                domains:
+                  type: object
+                  properties:
+                    exact:
+                      type: integer
+                      description: Number of exactly matching domains
+                      example: 1
+                    regex:
+                      type: integer
+                      description: Number of regex matching domains
+                      example: 2
+                gravity:
+                  type: object
+                  properties:
+                    allow:
+                      type: integer
+                      description: Number of allow matches (antigravity)
+                      example: 0
+                    block:
+                      type: integer
+                      description: Number of block matches (gravity)
+                      example: 1
+                total:
+                  type: integer
+                  description: Total number of matches
+                  example: 4
+  parameters:
+    domain:
+      name: domain
+      in: path
+      description: (Part of) domain to be searched for
+      schema:
+        type: string
+        default: ""
+        example: doubleclick.net
+      required: true
+    partial:
+      name: partial
+      in: query
+      description: Is partial matching requested?
+      required: false
+      schema:
+        type: boolean
+        default: false
+        example: false
+    N:
+      name: N
+      in: query
+      description: Maximum number of results to be returned
+      required: false
+      schema:
+        type: integer
+        default: 20
+        example: 20
+    debug:
+      name: debug
+      in: query
+      description: Add debug information to the response
+      required: false
+      schema:
+        type: boolean
+        default: false
+        example: false
+

src/api/docs/content/specs/stats.yaml

@@ -0,0 +1,789 @@
+openapi: 3.0.2
+components:
+  paths:
+    summary:
+      get:
+        summary: Get overview of Pi-hole activity
+        tags:
+          - Metrics
+        operationId: "get_metrics_summary"
+        description: |
+          Request various query, system, and FTL properties
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'stats.yaml#/components/schemas/queries'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+
+    database_summary:
+      get:
+        summary: Get database content details
+        tags:
+          - Metrics
+        operationId: "get_metrics_database_summary"
+        description: |
+          Request various database content details
+        parameters:
+          - $ref: 'common.yaml#/components/parameters/database/from'
+          - $ref: 'common.yaml#/components/parameters/database/until'
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'stats.yaml#/components/schemas/database_summary'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+
+    upstreams:
+      get:
+        summary: Get metrics about Pi-hole's upstream destinations
+        tags:
+          - Metrics
+        operationId: "get_metrics_upstreams"
+        description: |
+          Request upstream metrics
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'stats.yaml#/components/schemas/upstreams'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+
+    database_upstreams:
+      get:
+        summary: Get metrics about Pi-hole's upstream destinations (long-term database)
+        tags:
+          - Metrics
+        operationId: "get_metrics_upstreams_database"
+        description: |
+          Request upstream metrics (long-term database)
+        parameters:
+          - $ref: 'common.yaml#/components/parameters/database/from'
+          - $ref: 'common.yaml#/components/parameters/database/until'
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'stats.yaml#/components/schemas/upstreams'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    recent_blocked:
+      get:
+        summary: Get most recently blocked domain
+        tags:
+          - Metrics
+        operationId: "get_metrics_recent_blocked"
+        description: |
+          Request most recently blocked domain
+        parameters:
+          - $ref: 'stats.yaml#/components/parameters/recent_blocked/count'
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'stats.yaml#/components/schemas/recent_blocked'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    top_domains:
+      get:
+        summary: Get top domains
+        tags:
+          - Metrics
+        operationId: "get_metrics_top_domains"
+        description: |
+          Request top domains
+        parameters:
+          - $ref: 'stats.yaml#/components/parameters/top_items/blocked'
+          - $ref: 'stats.yaml#/components/parameters/top_items/count'
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'stats.yaml#/components/schemas/top_domains'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    database_top_domains:
+      get:
+        summary: Get top domains (long-term database)
+        tags:
+          - Metrics
+        operationId: "get_metrics_database_top_domains"
+        description: |
+          Request top domains
+        parameters:
+          - $ref: 'common.yaml#/components/parameters/database/from'
+          - $ref: 'common.yaml#/components/parameters/database/until'
+          - $ref: 'stats.yaml#/components/parameters/top_items/blocked'
+          - $ref: 'stats.yaml#/components/parameters/top_items/count'
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'stats.yaml#/components/schemas/top_domains'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    top_clients:
+      get:
+        summary: Get top clients
+        tags:
+          - Metrics
+        operationId: "get_metrics_top_clients"
+        description: |
+          Request top clients
+        parameters:
+          - $ref: 'stats.yaml#/components/parameters/top_items/blocked'
+          - $ref: 'stats.yaml#/components/parameters/top_items/count'
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'stats.yaml#/components/schemas/top_clients'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    database_top_clients:
+      get:
+        summary: Get top clients (long-term database)
+        tags:
+          - Metrics
+        operationId: "get_metrics_database_top_clients"
+        description: |
+          Request top clients
+        parameters:
+          - $ref: 'common.yaml#/components/parameters/database/from'
+          - $ref: 'common.yaml#/components/parameters/database/until'
+          - $ref: 'stats.yaml#/components/parameters/top_items/blocked'
+          - $ref: 'stats.yaml#/components/parameters/top_items/count'
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'stats.yaml#/components/schemas/top_clients'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    query_types:
+      get:
+        summary: Get query types
+        tags:
+          - Metrics
+        operationId: "get_metrics_query_types"
+        description: |
+          Request query types
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'stats.yaml#/components/schemas/query_types'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+    database_query_types:
+      get:
+        summary: Get query types (long-term database)
+        tags:
+          - Metrics
+        operationId: "get_metrics_database_query_types"
+        description: |
+          Request query types
+        parameters:
+          - $ref: 'common.yaml#/components/parameters/database/from'
+          - $ref: 'common.yaml#/components/parameters/database/until'
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'stats.yaml#/components/schemas/query_types'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+
+  schemas:
+    queries:
+      type: object
+      properties:
+        queries:
+          type: object
+          properties:
+            total:
+              type: integer
+              description: Total number of queries
+              example: 7497
+            blocked:
+              type: integer
+              description: Number of blocked queries
+              example: 3465
+            percent_blocked:
+              type: number
+              description: Percent of blocked queries
+              example: 34.5
+            unique_domains:
+              type: integer
+              description: Number of unique domains FTL knows
+              example: 445
+            forwarded:
+              type: integer
+              description: Number of queries that have been forwarded upstream
+              example: 4574
+            cached:
+              type: integer
+              description: Number of queries replied to from cache or local configuration
+              example: 9765
+            types:
+              type: object
+              description: Number of individual queries
+              properties:
+                A:
+                  type: integer
+                  description: Type A queries
+                  example: 3643
+                AAAA:
+                  type: integer
+                  description: Type AAAA queries
+                  example: 123
+                ANY:
+                  type: integer
+                  description: Type ANY queries
+                  example: 3423
+                SRV:
+                  type: integer
+                  description: Type SRV queries
+                  example: 345
+                SOA:
+                  type: integer
+                  description: Type SOA queries
+                  example: 7567
+                PTR:
+                  type: integer
+                  description: Type PTR queries
+                  example: 456
+                TXT:
+                  type: integer
+                  description: Type TXT queries
+                  example: 85
+                NAPTR:
+                  type: integer
+                  description: Type NAPTR queries
+                  example: 346
+                MX:
+                  type: integer
+                  description: Type MX queries
+                  example: 457
+                DS:
+                  type: integer
+                  description: Type DS queries
+                  example: 456
+                RRSIG:
+                  type: integer
+                  description: Type RRSIG queries
+                  example: 345
+                DNSKEY:
+                  type: integer
+                  description: Type DNSKEY queries
+                  example: 55
+                NS:
+                  type: integer
+                  description: Type NS queries
+                  example: 868
+                SVCB:
+                  type: integer
+                  description: Type SVCB queries
+                  example: 645
+                HTTPS:
+                  type: integer
+                  description: Type HTTPS queries
+                  example: 4
+                OTHER:
+                  type: integer
+                  description: Queries of remaining types
+                  example: 845
+            status:
+              type: object
+              description: Number of individual queries (by status)
+              properties:
+                UNKNOWN:
+                  type: integer
+                  description: Type UNKNOWN queries
+                  example: 3
+                GRAVITY:
+                  type: integer
+                  description: Type GRAVITY queries
+                  example: 72
+                FORWARDED:
+                  type: integer
+                  description: Type FORWARDED queries
+                  example: 533
+                CACHE:
+                  type: integer
+                  description: Type CACHE queries
+                  example: 32
+                REGEX:
+                  type: integer
+                  description: Type REGEX queries
+                  example: 84
+                DENYLIST:
+                  type: integer
+                  description: Type DENYLIST queries
+                  example: 31
+                EXTERNAL_BLOCKED_IP:
+                  type: integer
+                  description: Type EXTERNAL_BLOCKED_IP queries
+                  example: 0
+                EXTERNAL_BLOCKED_NULL:
+                  type: integer
+                  description: Type EXTERNAL_BLOCKED_NULL queries
+                  example: 0
+                EXTERNAL_BLOCKED_NXRA:
+                  type: integer
+                  description: Type EXTERNAL_BLOCKED_NXRA queries
+                  example: 0
+                GRAVITY_CNAME:
+                  type: integer
+                  description: Type GRAVITY_CNAME queries
+                  example: 0
+                REGEX_CNAME:
+                  type: integer
+                  description: Type REGEX_CNAME queries
+                  example: 0
+                DENYLIST_CNAME:
+                  type: integer
+                  description: Type DENYLIST_CNAME queries
+                  example: 0
+                RETRIED:
+                  type: integer
+                  description: Type RETRIED queries
+                  example: 0
+                RETRIED_DNSSEC:
+                  type: integer
+                  description: Type RETRIED_DNSSEC queries
+                  example: 0
+                IN_PROGRESS:
+                  type: integer
+                  description: Type IN_PROGRESS queries
+                  example: 0
+                DBBUSY:
+                  type: integer
+                  description: Type DBBUSY queries
+                  example: 0
+                SPECIAL_DOMAIN:
+                  type: integer
+                  description: Type SPECIAL_DOMAIN queries
+                  example: 0
+                CACHE_STALE:
+                  type: integer
+                  description: Type CACHE_STALE queries
+                  example: 0
+            replies:
+              type: object
+              description: Number of individual replies
+              properties:
+                UNKNOWN:
+                  type: integer
+                  description: Type UNKNOWN replies
+                  example: 3
+                NODATA:
+                  type: integer
+                  description: Type NODATA replies
+                  example: 72
+                NXDOMAIN:
+                  type: integer
+                  description: Type NXDOMAIN replies
+                  example: 533
+                CNAME:
+                  type: integer
+                  description: Type CNAME replies
+                  example: 32
+                IP:
+                  type: integer
+                  description: Type IP replies
+                  example: 84
+                DOMAIN:
+                  type: integer
+                  description: Type DOMAIN replies
+                  example: 31
+                RRNAME:
+                  type: integer
+                  description: Type RRNAME replies
+                  example: 0
+                SERVFAIL:
+                  type: integer
+                  description: Type SERVFAIL replies
+                  example: 0
+                REFUSED:
+                  type: integer
+                  description: Type REFUSED replies
+                  example: 0
+                NOTIMP:
+                  type: integer
+                  description: Type NOTIMP replies
+                  example: 0
+                OTHER:
+                  type: integer
+                  description: Type OTHER replies
+                  example: 0
+                DNSSEC:
+                  type: integer
+                  description: Type DNSSEC replies
+                  example: 31
+                NONE:
+                  type: integer
+                  description: Type NONE replies
+                  example: 0
+                BLOB:
+                  type: integer
+                  description: Type BLOB replies
+                  example: 0
+        clients:
+          type: object
+          properties:
+            active:
+              type: integer
+              description: Number of active clients (seen in the last 24 hours)
+              example: 10
+            total:
+              type: integer
+              description: Total number of clients seen by FTL
+              example: 22
+        gravity:
+          type: object
+          properties:
+            domains_being_blocked:
+              type: integer
+              description: Number of domain on your Pi-hole's gravity list
+              example: 104756
+    upstreams:
+      type: object
+      properties:
+        upstreams:
+          type: array
+          description: Array of upstream destinations
+          items:
+            type: object
+            properties:
+              ip:
+                type: string
+                description: Upstream destination's IP address (can be either IPv4 or IPv6)
+                nullable: true
+                example: "127.0.0.1"
+              name:
+                type: string
+                description: Upstream destination's hostname (if available)
+                nullable: true
+                example: "localhost"
+              port:
+                type: integer
+                description: Upstream destination's destination port (-1 if not applicable, e.g., for the local cache)
+                example: 53
+              count:
+                type: integer
+                description: Number of queries this upstream destination has been used for
+                example: 65445
+              statistics:
+                type: object
+                properties:
+                  response:
+                    type: number
+                    description: Average response time of this upstream destination in seconds (0 if not applicable)
+                    example: 0.0254856
+                  variance:
+                    type: number
+                    description: Standard deviation of the average response time (0 if not applicable)
+                    example: 0.02058
+        forwarded_queries:
+          type: integer
+          description: Number of forwarded queries
+          example: 6379
+        total_queries:
+          type: integer
+          description: Total number of queries
+          example: 29160
+    top_domains:
+      type: object
+      properties:
+        domains:
+          type: array
+          description: Array of domains
+          items:
+            type: object
+            properties:
+              domain:
+                type: string
+                description: Requested domain
+                example: "pi-hole.net"
+              count:
+                type: integer
+                description: Number of times this domain has been requested
+                example: 8516
+        total_queries:
+          type: integer
+          description: Total number of queries
+          example: 29160
+        blocked_queries:
+          type: integer
+          description: Number of blocked queries
+          example: 6379
+    top_clients:
+      type: object
+      properties:
+        clients:
+          type: array
+          description: Array of clients
+          items:
+            type: object
+            properties:
+              ip:
+                type: string
+                description: Client IP address (can be either IPv4 or IPv6)
+                example: "192.168.0.44"
+              name:
+                type: string
+                description: Client hostname (if available)
+                example: "raspberrypi.lan"
+              count:
+                type: integer
+                description: Number of queries this client has made
+                example: 5896
+        total_queries:
+          type: integer
+          description: Total number of queries
+          example: 29160
+        blocked_queries:
+          type: integer
+          description: Number of blocked queries
+          example: 6379
+    query_types:
+      type: object
+      properties:
+        types:
+          type: object
+          description: Number of individual query types
+          properties:
+            A:
+              type: integer
+              description: Type A queries
+              example: 18268
+            AAAA:
+              type: integer
+              description: Type AAAA queries
+              example: 2332
+            ANY:
+              type: integer
+              description: Type ANY queries
+              example: 0
+            SRV:
+              type: integer
+              description: Type SRV queries
+              example: 6
+            SOA:
+              type: integer
+              description: Type SOA queries
+              example: 44
+            PTR:
+              type: integer
+              description: Type PTR queries
+              example: 389
+            TXT:
+              type: integer
+              description: Type TXT queries
+              example: 0
+            NAPTR:
+              type: integer
+              description: Type NAPTR queries
+              example: 1
+            MX:
+              type: integer
+              description: Type MX queries
+              example: 109
+            DS:
+              type: integer
+              description: Type DS queries
+              example: 596
+            RRSIG:
+              type: integer
+              description: Type RRSIG queries
+              example: 25
+            DNSKEY:
+              type: integer
+              description: Type DNSKEY queries
+              example: 240
+            NS:
+              type: integer
+              description: Type NS queries
+              example: 18
+            SVCB:
+              type: integer
+              description: Type SVCB queries
+              example: 0
+            HTTPS:
+              type: integer
+              description: Type HTTPS queries
+              example: 11
+            OTHER:
+              type: integer
+              description: Type OTHER queries
+              example: 0
+    database_summary:
+      type: object
+      properties:
+        sum_queries:
+          type: integer
+          description: Total number of queries
+          example: 29160
+        sum_blocked:
+          type: integer
+          description: Total number of blocked queries
+          example: 6379
+        percent_blocked:
+          type: number
+          description: Percentage of blocked queries
+          example: 21.9
+        total_clients:
+          type: integer
+          description: Total number of clients
+          example: 10
+    recent_blocked:
+      type: object
+      properties:
+        blocked:
+          type: array
+          description: List of blocked domains
+          items:
+            type: string
+          example: ["doubleclick.net"]
+
+  parameters:
+    recent_blocked:
+      count:
+        in: query
+        description: Number of requested blocked domains
+        name: count
+        schema:
+          type: integer
+        required: false
+        example: 1
+    top_items:
+      blocked:
+        in: query
+        description: Return information about permitted or blocked queries
+        name: blocked
+        schema:
+          type: boolean
+        required: false
+        example: false
+      count:
+        in: query
+        description: Number of requested items
+        name: count
+        schema:
+          type: integer
+        required: false
+        example: 10

src/api/docs/content/specs/teleporter.yaml

@@ -0,0 +1,155 @@
+openapi: 3.0.2
+components:
+  paths:
+    teleporter:
+      get:
+        summary: Export Pi-hole settings
+        tags:
+          - "Pi-hole configuration"
+        operationId: "get_teleporter"
+        description: |
+          Request an archived copy of your Pi-hole's current configuration.
+          Authentication via header or cookie is required for the endpoint.
+        responses:
+          '200':
+            description: OK
+            content:
+              application/zip:
+                schema:
+                  type: string
+                  format: binary
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+      post:
+        summary: Import Pi-hole settings
+        tags:
+          - "Pi-hole configuration"
+        operationId: "post_teleporter"
+        description: |
+          Upload a Pi-hole Teleporter archive to (partially) restore from it. Note that this will overwrite your current configuration.
+          Authentication via header or cookie is required for the endpoint.
+        requestBody:
+          content:
+            multipart/form-data:
+              schema:
+                properties:
+                  file:
+                    type: string
+                    format: binary
+                  import:
+                    type: object
+                    nullable: true
+                    properties:
+                      config:
+                        type: boolean
+                        description: "Import Pi-hole configuration"
+                        example: true
+                      dhcp_leases:
+                        type: boolean
+                        description: "Import Pi-hole DHCP leases"
+                        example: true
+                      gravity:
+                        type: object
+                        properties:
+                          group:
+                            type: boolean
+                            description: "Import Pi-hole's groups table"
+                            example: true
+                          adlist:
+                            type: boolean
+                            description: "Import Pi-hole's adlist table"
+                            example: true
+                          adlist_by_group:
+                            type: boolean
+                            description: "Import Pi-hole's table relating adlist entries to groups"
+                            example: true
+                          domainlist:
+                            type: boolean
+                            description: "Import Pi-hole's domainlist table"
+                            example: true
+                          domainlist_by_group:
+                            type: boolean
+                            description: "Import Pi-hole's table relating domainlist entries to groups"
+                            example: true
+                          client:
+                            type: boolean
+                            description: "Import Pi-hole's client table"
+                            example: true
+                          client_by_group:
+                            type: boolean
+                            description: "Import Pi-hole's table relating client entries to groups"
+                            example: true
+                    description: "A JSON object of files to import. If omitted, all files will be imported."
+        responses:
+          '200':
+            description: OK
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'teleporter.yaml#/components/schemas/teleporter/post'
+                    - $ref: 'common.yaml#/components/schemas/took'
+                examples:
+                  teleporter:
+                    $ref: 'teleporter.yaml#/components/examples/teleporter'
+          '400':
+            description: Bad request
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'teleporter.yaml#/components/errors/invalid_zip'
+                    - $ref: 'common.yaml#/components/schemas/took'
+          '401':
+            description: Unauthorized
+            content:
+              application/json:
+                schema:
+                  allOf:
+                    - $ref: 'common.yaml#/components/errors/unauthorized'
+                    - $ref: 'common.yaml#/components/schemas/took'
+  schemas:
+    teleporter:
+      post:
+        type: object
+        properties:
+          processed:
+            type: array
+            items:
+              type: string
+  errors:
+    invalid_zip:
+      type: object
+      description: Invalid ZIP archive uploaded
+      properties:
+        error:
+          type: object
+          properties:
+            key:
+              type: string
+              description: "Machine-readable error type"
+              example: "invalid_zip"
+            message:
+              type: string
+              description: "Human-readable error message"
+              example: "Invalid ZIP file uploaded"
+            hint:
+              type: string
+              nullable: true
+              description: "No additional data available"
+              example: null
+
+  examples:
+    teleporter:
+      value:
+        processed:
+          - etc/pihole/pihole.toml
+          - etc/pihole/gravity.db->group
+          - etc/pihole/gravity.db->adlist
+          - etc/pihole/gravity.db->adlist_by_group

src/api/docs/docs.c

@@ -0,0 +1,44 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2021 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  API Implementation /api/docs
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#include "docs.h"
+
+int api_docs(struct ftl_conn *api)
+{
+	// Handle resource request by redirecting to "/"
+	if(strcmp(api->request->request_uri, "/api/docs") == 0)
+	{
+		log_debug(DEBUG_API, "Redirecting /api/docs --301--> /api/docs/");
+		mg_send_http_redirect(api->conn, "/api/docs/", 301);
+	}
+
+	// Handle root request by redirecting to "/"
+	bool serve_index = false;
+	if(strcmp(api->request->request_uri, "/api/docs/") == 0)
+	{
+		serve_index = true;
+	}
+
+	// Loop over all available files and see if we can serve this request
+	for(unsigned int i = 0; i < (sizeof(docs_files)/sizeof(docs_files[0])); i++)
+	{
+		// Check if this is the requested file
+		if(strcmp(docs_files[i].path, api->item) == 0 ||
+		   (serve_index && strcmp(docs_files[i].path, "index.html") == 0))
+		{
+			// Send the file
+			mg_send_http_ok(api->conn, docs_files[i].mime_type, (long long)docs_files[i].content_size);
+			return mg_write(api->conn, docs_files[i].content, docs_files[i].content_size);
+		}
+	}
+
+	// Requested path was not found
+	return 0;
+}

src/api/docs/docs.h

@@ -0,0 +1,173 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2021 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  API Implementation /api/docs (helper)
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+#ifndef API_DOCS_H
+#define API_DOCS_H
+
+#include "FTL.h"
+#include "webserver/civetweb/civetweb.h"
+#include "webserver/http-common.h"
+#include "webserver/json_macros.h"
+#include "api/api.h"
+
+static const unsigned char index_html[] = {
+#include "hex/index.html"
+};
+
+static const unsigned char index_css[] = {
+#include "hex/index.css"
+};
+
+static const unsigned char pi_hole_js[] = {
+#include "hex/pi-hole.js"
+};
+
+static const unsigned char rapidoc_min_js[] = {
+#include "hex/external/rapidoc-min.js"
+};
+
+static const unsigned char rapidoc_min_js_map[] = {
+#include "hex/external/rapidoc-min.js.map"
+};
+
+static const unsigned char highlight_default_min_css[] = {
+#include "hex/external/highlight-default.min.css"
+};
+
+static const unsigned char highlight_min_js[] = {
+#include "hex/external/highlight.min.js"
+};
+
+static const unsigned char images_logo_svg[] = {
+#include "hex/images/logo.svg"
+};
+
+static const unsigned char specs_auth_yaml[] = {
+#include "hex/specs/auth.yaml"
+};
+
+static const unsigned char specs_clients_yaml[] = {
+#include "hex/specs/clients.yaml"
+};
+
+static const unsigned char specs_common_yaml[] = {
+#include "hex/specs/common.yaml"
+};
+
+static const unsigned char specs_dhcp_yaml[] = {
+#include "hex/specs/dhcp.yaml"
+};
+
+static const unsigned char specs_dns_yaml[] = {
+#include "hex/specs/dns.yaml"
+};
+
+static const unsigned char specs_docs_yaml[] = {
+#include "hex/specs/docs.yaml"
+};
+
+static const unsigned char specs_domains_yaml[] = {
+#include "hex/specs/domains.yaml"
+};
+
+static const unsigned char specs_info_yaml[] = {
+#include "hex/specs/info.yaml"
+};
+
+static const unsigned char specs_groups_yaml[] = {
+#include "hex/specs/groups.yaml"
+};
+
+static const unsigned char specs_history_yaml[] = {
+#include "hex/specs/history.yaml"
+};
+
+static const unsigned char specs_lists_yaml[] = {
+#include "hex/specs/lists.yaml"
+};
+
+static const unsigned char specs_main_yaml[] = {
+#include "hex/specs/main.yaml"
+};
+
+static const unsigned char specs_queries_yaml[] = {
+#include "hex/specs/queries.yaml"
+};
+
+static const unsigned char specs_stats_yaml[] = {
+#include "hex/specs/stats.yaml"
+};
+
+static const unsigned char specs_config_yaml[] = {
+#include "hex/specs/config.yaml"
+};
+
+static const unsigned char specs_network_yaml[] = {
+#include "hex/specs/network.yaml"
+};
+
+static const unsigned char specs_logs_yaml[] = {
+#include "hex/specs/logs.yaml"
+};
+
+static const unsigned char specs_endpoints_yaml[] = {
+#include "hex/specs/endpoints.yaml"
+};
+
+static const unsigned char specs_teleporter_yaml[] = {
+#include "hex/specs/teleporter.yaml"
+};
+
+static const unsigned char specs_search_yaml[] = {
+#include "hex/specs/search.yaml"
+};
+
+static const unsigned char specs_action_yaml[] = {
+#include "hex/specs/action.yaml"
+};
+
+struct {
+    const char *path;
+    const char *mime_type;
+    const char *content;
+    const size_t content_size;
+} docs_files[] =
+{
+    {"index.html", "text/html", (const char*)index_html, sizeof(index_html)},
+    {"index.css", "text/css", (const char*)index_css, sizeof(index_css)},
+    {"pi-hole.js", "application/javascript", (const char*)pi_hole_js, sizeof(pi_hole_js)},
+    {"external/rapidoc-min.js", "application/javascript", (const char*)rapidoc_min_js, sizeof(rapidoc_min_js)},
+    {"external/rapidoc-min.js.map", "text/plain", (const char*)rapidoc_min_js_map, sizeof(rapidoc_min_js_map)},
+    {"external/highlight-default.min.css", "text/css", (const char*)highlight_default_min_css, sizeof(highlight_default_min_css)},
+    {"external/highlight.min.js", "application/javascript", (const char*)highlight_min_js, sizeof(highlight_min_js)},
+    {"images/logo.svg", "image/svg+xml", (const char*)images_logo_svg, sizeof(images_logo_svg)},
+    {"specs/auth.yaml", "text/plain", (const char*)specs_auth_yaml, sizeof(specs_auth_yaml)},
+    {"specs/clients.yaml", "text/plain", (const char*)specs_clients_yaml, sizeof(specs_clients_yaml)},
+    {"specs/config.yaml", "text/plain", (const char*)specs_config_yaml, sizeof(specs_config_yaml)},
+    {"specs/common.yaml", "text/plain", (const char*)specs_common_yaml, sizeof(specs_common_yaml)},
+    {"specs/dhcp.yaml", "text/plain", (const char*)specs_dhcp_yaml, sizeof(specs_dhcp_yaml)},
+    {"specs/dns.yaml", "text/plain", (const char*)specs_dns_yaml, sizeof(specs_dns_yaml)},
+    {"specs/domains.yaml", "text/plain", (const char*)specs_domains_yaml, sizeof(specs_domains_yaml)},
+    {"specs/docs.yaml", "text/plain", (const char*)specs_docs_yaml, sizeof(specs_docs_yaml)},
+    {"specs/endpoints.yaml", "text/plain", (const char*)specs_endpoints_yaml, sizeof(specs_endpoints_yaml)},
+    {"specs/groups.yaml", "text/plain", (const char*)specs_groups_yaml, sizeof(specs_groups_yaml)},
+    {"specs/history.yaml", "text/plain", (const char*)specs_history_yaml, sizeof(specs_history_yaml)},
+    {"specs/info.yaml", "text/plain", (const char*)specs_info_yaml, sizeof(specs_info_yaml)},
+    {"specs/lists.yaml", "text/plain", (const char*)specs_lists_yaml, sizeof(specs_lists_yaml)},
+    {"specs/logs.yaml", "text/plain", (const char*)specs_logs_yaml, sizeof(specs_logs_yaml)},
+    {"specs/main.yaml", "text/plain", (const char*)specs_main_yaml, sizeof(specs_main_yaml)},
+    {"specs/network.yaml", "text/plain", (const char*)specs_network_yaml, sizeof(specs_network_yaml)},
+    {"specs/queries.yaml", "text/plain", (const char*)specs_queries_yaml, sizeof(specs_queries_yaml)},
+    {"specs/search.yaml", "text/plain", (const char*)specs_search_yaml, sizeof(specs_search_yaml)},
+    {"specs/stats.yaml", "text/plain", (const char*)specs_stats_yaml, sizeof(specs_stats_yaml)},
+    {"specs/teleporter.yaml", "text/plain", (const char*)specs_teleporter_yaml, sizeof(specs_teleporter_yaml)},
+    {"specs/action.yaml", "text/plain", (const char*)specs_action_yaml, sizeof(specs_action_yaml)},
+};
+
+#endif // API_DOCS_H

src/api/history.c

@@ -0,0 +1,264 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2021 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  API Implementation
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#include "FTL.h"
+#include "webserver/http-common.h"
+#include "webserver/json_macros.h"
+#include "api.h"
+#include "shmem.h"
+#include "datastructure.h"
+// overTime data
+#include "overTime.h"
+// config struct
+#include "config/config.h"
+// read_setupVarsconf()
+#include "config/setupVars.h"
+// get_aliasclient_list()
+#include "database/aliasclients.h"
+
+int api_history(struct ftl_conn *api)
+{
+	lock_shm();
+
+	// Loop over all overTime slots and add them to the array
+	cJSON *history = JSON_NEW_ARRAY();
+	for(unsigned int slot = 0; slot < OVERTIME_SLOTS; slot++)
+	{
+		cJSON *item = JSON_NEW_OBJECT();
+		JSON_ADD_NUMBER_TO_OBJECT(item, "timestamp", overTime[slot].timestamp);
+		JSON_ADD_NUMBER_TO_OBJECT(item, "total", overTime[slot].total);
+		JSON_ADD_NUMBER_TO_OBJECT(item, "cached", overTime[slot].cached);
+		JSON_ADD_NUMBER_TO_OBJECT(item, "blocked", overTime[slot].blocked);
+		JSON_ADD_NUMBER_TO_OBJECT(item, "forwarded", overTime[slot].forwarded);
+		JSON_ADD_ITEM_TO_ARRAY(history, item);
+	}
+
+	// Unlock already here to avoid keeping the lock during JSON generation
+	// This is safe because we don't access any shared memory after this
+	// point. All numbers in the JSON are copied
+	unlock_shm();
+
+	// Minimum structure is
+	// {"history":[]}
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_ADD_ITEM_TO_OBJECT(json, "history", history);
+	JSON_SEND_OBJECT(json);
+}
+
+static unsigned int build_client_temparray(int *temparray, const int slot)
+{
+	// Clear temporary array
+	memset(temparray, 0, 2 * counters->clients * sizeof(int));
+
+	unsigned int num_clients = 0;
+	for(int clientID = 0; clientID < counters->clients; clientID++)
+	{
+		// Get client pointer
+		const clientsData* client = getClient(clientID, true);
+
+		// Skip invalid (recycled) clients
+		if(client == NULL)
+			continue;
+
+		// If this client is managed by an alias-client, we substitute
+		// -1 for the total count
+		if(!client->flags.aliasclient && client->aliasclient_id > -1)
+		{
+			log_debug(DEBUG_API, "Skipping client (ID %d) contained in alias-client with ID %d",
+			          clientID, client->aliasclient_id);
+			continue;
+		}
+		else
+
+		// Store clientID and number of queries in temporary array
+		// If the slot is -1, we return the total number of queries.
+		// Otherwise, we return the number of queries in the given time
+		// slot
+		temparray[2*num_clients + 0] = clientID;
+		temparray[2*num_clients + 1] = slot < 0 ? client->count : client->overTime[slot];
+
+		// Increase number of clients by one
+		num_clients++;
+	}
+
+	return num_clients;
+}
+
+int api_history_clients(struct ftl_conn *api)
+{
+	// Exit before processing any data if requested via config setting
+	if(config.misc.privacylevel.v.privacy_level >= PRIVACY_HIDE_DOMAINS_CLIENTS)
+	{
+		// Minimum structure is
+		// {"history":[], "clients":[]}
+		cJSON *json = JSON_NEW_OBJECT();
+		cJSON *history = JSON_NEW_ARRAY();
+		JSON_ADD_ITEM_TO_OBJECT(json, "history", history);
+		cJSON *clients = JSON_NEW_ARRAY();
+		JSON_ADD_ITEM_TO_OBJECT(json, "clients", clients);
+		JSON_SEND_OBJECT_UNLOCK(json);
+	}
+
+	// Get number of clients to return
+	unsigned int Nc = min(counters->clients, config.webserver.api.maxClients.v.u16);
+	if(api->request->query_string != NULL)
+	{
+		// Does the user request a non-default number of clients
+		get_uint_var(api->request->query_string, "N", &Nc);
+	}
+
+	// Limit the number of clients to the maximum number of clients
+	if(Nc == 0 || Nc > (unsigned int)counters->clients)
+	{
+		// Return all clients
+		Nc = counters->clients;
+	}
+
+	// Lock shared memory
+	lock_shm();
+
+	// Allocate memory for the temporary buffer for ranking our clients
+	int *temparray = calloc(counters->clients, 2 * sizeof(int));
+	if(temparray == NULL)
+	{
+		unlock_shm();
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to allocate memory for temporary array",
+		                       NULL);
+	}
+
+	// Get MAX_CLIENTS clients with the highest number of queries
+	// Skip clients included in others (in alias-clients)
+	unsigned int num_clients = build_client_temparray(temparray, -1);
+
+	if(config.webserver.api.client_history_global_max.v.b)
+	{
+		// Sort temporary array. Even when the array itself has <counters.clients>
+		// elements, we only sort the first <clients> elements to avoid sorting
+		// the whole array (the final elements are not used when clients have been
+		// skipped above, e.g. alias-clients or recycled clients)
+		qsort(temparray, num_clients, sizeof(int[2]), cmpdesc);
+	}
+
+	// Main return loop
+	int others_total = 0;
+
+	cJSON *history = JSON_NEW_ARRAY();
+	for(unsigned int slot = 0; slot < OVERTIME_SLOTS; slot++)
+	{
+		cJSON *item = JSON_NEW_OBJECT();
+		JSON_ADD_NUMBER_TO_OBJECT(item, "timestamp", overTime[slot].timestamp);
+
+		// If we are not in global-max mode, we need to build the temporary
+		// client array for each slot individually
+		if(!config.webserver.api.client_history_global_max.v.b)
+		{
+			// Collect global client data
+			num_clients = build_client_temparray(temparray, slot);
+
+			// Sort temporary array. Even when the array itself has <counters.clients>
+			// elements, we only sort the first <clients> elements to avoid sorting
+			// the whole array (the final elements are not used when clients have been
+			// skipped above, e.g. alias-clients or recycled clients)
+			qsort(temparray, num_clients, sizeof(int[2]), cmpdesc);
+		}
+
+		// Loop over clients to generate output to be sent to the client
+		int others = 0;
+		cJSON *data = JSON_NEW_OBJECT();
+		for(unsigned int arrayID = 0; arrayID < num_clients; arrayID++)
+		{
+
+			// Get client pointer
+			const int clientID = temparray[2*arrayID + 0];
+
+			// All clientIDs will be valid because we only added
+			// valid clients to the temparray
+			const clientsData* client = getClient(clientID, true);
+
+			// Skip further clients when we reached the maximum
+			// number of clients to return They are summed together
+			// under the special "other" client
+			// -1 because of the special "other" client we add below
+			// This is disabled when Nc is 0, which means we want to return
+			// all clients
+			if(arrayID >= Nc - 1)
+			{
+				others += client->overTime[slot];
+				continue;
+			}
+
+			// Add client to the array
+			cJSON_AddNumberToObject(data, getstr(client->ippos), client->overTime[slot]);
+		}
+		// Add others as last element in the array
+		others_total += others;
+		JSON_ADD_NUMBER_TO_OBJECT(data, "others", others);
+
+		JSON_ADD_ITEM_TO_OBJECT(item, "data", data);
+		JSON_ADD_ITEM_TO_ARRAY(history, item);
+	}
+
+	// Loop over clients to generate output to be sent to the client
+	cJSON *clients = JSON_NEW_OBJECT();
+	for(unsigned int arrayID = 0; arrayID < num_clients; arrayID++)
+	{
+		// Get client pointer
+		const int clientID = temparray[2*arrayID + 0];
+
+		// All clientIDs will be valid because we only added
+		// valid clients to the temparray
+		const clientsData* client = getClient(clientID, true);
+
+		// Break once we reached the maximum number of clients to return
+		// -1 because of the special "other" client we add below
+		// This is disabled when
+		// - N is 0, which means we want to return all clients, or
+		// - when we are NOT in global-max mode as we need to return all
+		//   clients in that case
+		if(config.webserver.api.client_history_global_max.v.b && arrayID >= Nc - 1)
+			break;
+
+		// Get client name and IP address
+		const char *client_ip = getstr(client->ippos);
+		const char *client_name = client->namepos != 0 ? getstr(client->namepos) : NULL;
+
+		// Create JSON object for this client
+		cJSON *item = JSON_NEW_OBJECT();
+		JSON_REF_STR_IN_OBJECT(item, "name", client_name);
+		JSON_ADD_NUMBER_TO_OBJECT(item, "total", client->count);
+		JSON_ADD_ITEM_TO_OBJECT(clients, client_ip, item);
+	}
+
+	// Unlock already here to avoid keeping the lock during JSON generation
+	// This is safe because we don't access any shared memory after this
+	// point and all strings in the JSON are references to idempotent shared
+	// memory and can, thus, be accessed at any time without locking
+	unlock_shm();
+
+	// Add "others" client only if there are more clients than we return
+	// and if we are not returning all clients
+	if(num_clients > Nc)
+	{
+		cJSON *item = JSON_NEW_OBJECT();
+		JSON_REF_STR_IN_OBJECT(item, "name", "other clients");
+		JSON_ADD_NUMBER_TO_OBJECT(item, "total", others_total);
+		JSON_ADD_ITEM_TO_OBJECT(clients, "others", item);
+	}
+
+	// Free memory
+	free(temparray);
+
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_ADD_ITEM_TO_OBJECT(json, "history", history);
+	JSON_ADD_ITEM_TO_OBJECT(json, "clients", clients);
+	JSON_SEND_OBJECT(json);
+}

src/api/list.c

@@ -0,0 +1,936 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2020 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  API Implementation /api/{allow,deny}list
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#include "FTL.h"
+#include "webserver/http-common.h"
+#include "webserver/json_macros.h"
+#include "api.h"
+#include "database/gravity-db.h"
+#include "events.h"
+#include "shmem.h"
+// getNameFromIP()
+#include "database/network-table.h"
+// valid_domain()
+#include "tools/gravity-parseList.h"
+// parse_groupIDs()
+#include "webserver/http-common.h"
+#include <idn2.h>
+
+static int api_list_read(struct ftl_conn *api,
+                         const int code,
+                         const enum gravity_list_type listtype,
+                         const char *item,
+                         cJSON *processed)
+{
+	const char *sql_msg = NULL;
+	if(!gravityDB_readTable(listtype, item, &sql_msg, true, NULL))
+	{
+		return send_json_error(api, 400, // 400 Bad Request
+		                       "database_error",
+		                       "Could not read domains from database table",
+		                       sql_msg);
+	}
+
+	tablerow table = { 0 };
+	cJSON *rows = JSON_NEW_ARRAY();
+	while(gravityDB_readTableGetRow(listtype, &table, &sql_msg))
+	{
+		cJSON *row = JSON_NEW_OBJECT();
+
+		// Special fields
+		if(listtype == GRAVITY_GROUPS)
+		{
+			JSON_COPY_STR_TO_OBJECT(row, "name", table.name);
+			JSON_COPY_STR_TO_OBJECT(row, "comment", table.comment);
+		}
+		else if(listtype == GRAVITY_ADLISTS ||
+		        listtype == GRAVITY_ADLISTS_BLOCK ||
+		        listtype == GRAVITY_ADLISTS_ALLOW)
+		{
+			JSON_COPY_STR_TO_OBJECT(row, "address", table.address);
+			JSON_COPY_STR_TO_OBJECT(row, "comment", table.comment);
+		}
+		else if(listtype == GRAVITY_CLIENTS)
+		{
+			char *name = NULL;
+			if(table.client != NULL)
+			{
+				// Try to obtain hostname
+				if(isValidIPv4(table.client) || isValidIPv6(table.client))
+					name = getNameFromIP(NULL, table.client);
+				else if(isMAC(table.client))
+					name = getNameFromMAC(table.client);
+			}
+
+			JSON_COPY_STR_TO_OBJECT(row, "client", table.client);
+			JSON_COPY_STR_TO_OBJECT(row, "name", name);
+			JSON_COPY_STR_TO_OBJECT(row, "comment", table.comment);
+
+			// Free allocated memory (if applicable)
+			if(name != NULL)
+				free(name);
+		}
+		else // domainlists
+		{
+			char *unicode = NULL;
+			const int rc = idn2_to_unicode_lzlz(table.domain, &unicode, IDN2_NONTRANSITIONAL);
+			JSON_COPY_STR_TO_OBJECT(row, "domain", table.domain);
+			if(rc == IDN2_OK)
+				JSON_COPY_STR_TO_OBJECT(row, "unicode", unicode);
+			else
+				JSON_COPY_STR_TO_OBJECT(row, "unicode", table.domain);
+			JSON_REF_STR_IN_OBJECT(row, "type", table.type);
+			JSON_REF_STR_IN_OBJECT(row, "kind", table.kind);
+			JSON_COPY_STR_TO_OBJECT(row, "comment", table.comment);
+			if(unicode != NULL)
+				free(unicode);
+		}
+
+		// Groups don't have the groups property
+		if(listtype != GRAVITY_GROUPS)
+		{
+			if(table.group_ids != NULL)
+			{
+				const int ret = parse_groupIDs(api, &table, row);
+				if(ret != 0)
+				{
+					JSON_DELETE(rows);
+					return ret;
+				}
+
+			}
+			else
+			{
+				// Empty group set
+				cJSON *group_ids = JSON_NEW_ARRAY();
+				JSON_ADD_ITEM_TO_OBJECT(row, "groups", group_ids);
+			}
+		}
+
+		// Clients don't have the enabled property
+		if(listtype != GRAVITY_CLIENTS)
+			JSON_ADD_BOOL_TO_OBJECT(row, "enabled", table.enabled);
+
+		// Add read-only database parameters
+		JSON_ADD_NUMBER_TO_OBJECT(row, "id", table.id);
+		JSON_ADD_NUMBER_TO_OBJECT(row, "date_added", table.date_added);
+		JSON_ADD_NUMBER_TO_OBJECT(row, "date_modified", table.date_modified);
+
+		// Properties added in https://github.com/pi-hole/pi-hole/pull/3951
+		if(listtype == GRAVITY_ADLISTS ||
+		   listtype == GRAVITY_ADLISTS_BLOCK ||
+		   listtype == GRAVITY_ADLISTS_ALLOW)
+		{
+			JSON_REF_STR_IN_OBJECT(row, "type", table.type);
+			JSON_ADD_NUMBER_TO_OBJECT(row, "date_updated", table.date_updated);
+			JSON_ADD_NUMBER_TO_OBJECT(row, "number", table.number);
+			JSON_ADD_NUMBER_TO_OBJECT(row, "invalid_domains", table.invalid_domains);
+			JSON_ADD_NUMBER_TO_OBJECT(row, "abp_entries", table.abp_entries);
+			JSON_ADD_NUMBER_TO_OBJECT(row, "status", table.status);
+		}
+
+		JSON_ADD_ITEM_TO_ARRAY(rows, row);
+	}
+	gravityDB_readTableFinalize();
+
+	if(sql_msg == NULL)
+	{
+		// No error, send domains array
+		const char *objname;
+		cJSON *json = JSON_NEW_OBJECT();
+		if(listtype == GRAVITY_GROUPS)
+			objname = "groups";
+		else if(listtype == GRAVITY_ADLISTS ||
+		        listtype == GRAVITY_ADLISTS_BLOCK ||
+		        listtype == GRAVITY_ADLISTS_ALLOW)
+			objname = "lists";
+		else if(listtype == GRAVITY_CLIENTS)
+			objname = "clients";
+		else // domainlists
+			objname = "domains";
+		JSON_ADD_ITEM_TO_OBJECT(json, objname, rows);
+
+		// Add processed count (if applicable)
+		if(processed != NULL)
+			JSON_ADD_ITEM_TO_OBJECT(json, "processed", processed);
+
+		JSON_SEND_OBJECT_CODE(json, code);
+	}
+	else
+	{
+		JSON_DELETE(rows);
+		return send_json_error(api, 400, // 400 Bad Request
+		                       "database_error",
+		                       "Could not read from gravity database",
+		                       sql_msg);
+	}
+}
+
+static int api_list_write(struct ftl_conn *api,
+                          const enum gravity_list_type listtype,
+                          const char *item)
+{
+	tablerow row = { 0 };
+
+	// Check if valid JSON payload is available
+	const int json_ret = check_json_payload(api);
+	if(json_ret != 0)
+		return json_ret;
+
+	bool spaces_allowed = false;
+	bool allocated_json = false;
+	if(api->method == HTTP_POST)
+	{
+		// Extract domain/name/client/address from payload when using POST, all
+		// others specify it as URI-component
+		switch(listtype)
+		{
+			case GRAVITY_DOMAINLIST_ALLOW_EXACT:
+			case GRAVITY_DOMAINLIST_ALLOW_REGEX:
+			case GRAVITY_DOMAINLIST_DENY_EXACT:
+			case GRAVITY_DOMAINLIST_DENY_REGEX:
+			{
+				cJSON* json_domain = cJSON_GetObjectItemCaseSensitive(api->payload.json, "domain");
+				if(cJSON_IsString(json_domain) && strlen(json_domain->valuestring) > 0)
+				{
+					row.items = cJSON_CreateArray();
+					cJSON_AddItemToArray(row.items, cJSON_CreateStringReference(json_domain->valuestring));
+					allocated_json = true;
+				}
+				else if(cJSON_IsArray(json_domain) && cJSON_GetArraySize(json_domain) > 0)
+					row.items = json_domain;
+				else
+				{
+					return send_json_error(api, 400,
+					                       "bad_request",
+					                       "Invalid request: No valid \"domain\" in payload (must be either string or array)",
+					                       NULL);
+				}
+				break;
+			}
+
+			case GRAVITY_GROUPS:
+			{
+				spaces_allowed = true;
+				cJSON *json_name = cJSON_GetObjectItemCaseSensitive(api->payload.json, "name");
+				if(cJSON_IsString(json_name) && strlen(json_name->valuestring) > 0)
+				{
+					row.items = cJSON_CreateArray();
+					cJSON_AddItemToArray(row.items, cJSON_CreateStringReference(json_name->valuestring));
+					allocated_json = true;
+				}
+				else if(cJSON_IsArray(json_name) && cJSON_GetArraySize(json_name) > 0)
+					row.items = json_name;
+				else
+				{
+					return send_json_error(api, 400,
+					                       "bad_request",
+					                       "Invalid request: No valid \"name\" in payload (must be either string or array)",
+					                       NULL);
+				}
+				break;
+			}
+
+			case GRAVITY_CLIENTS:
+			{
+				cJSON *json_client = cJSON_GetObjectItemCaseSensitive(api->payload.json, "client");
+				if(cJSON_IsString(json_client) && strlen(json_client->valuestring) > 0)
+				{
+					row.items = cJSON_CreateArray();
+					cJSON_AddItemToArray(row.items, cJSON_CreateStringReference(json_client->valuestring));
+					allocated_json = true;
+				}
+				else if(cJSON_IsArray(json_client) && cJSON_GetArraySize(json_client) > 0)
+					row.items = json_client;
+				else
+				{
+					return send_json_error(api, 400,
+					                       "bad_request",
+					                       "Invalid request: No valid \"client\" in payload (must be either string or array)",
+					                       NULL);
+				}
+				break;
+			}
+
+			case GRAVITY_ADLISTS:
+			case GRAVITY_ADLISTS_BLOCK:
+			case GRAVITY_ADLISTS_ALLOW:
+			{
+				cJSON *json_address = cJSON_GetObjectItemCaseSensitive(api->payload.json, "address");
+				if(cJSON_IsString(json_address) && strlen(json_address->valuestring) > 0)
+				{
+					row.items = cJSON_CreateArray();
+					cJSON_AddItemToArray(row.items,  cJSON_CreateStringReference(json_address->valuestring));
+					allocated_json = true;
+				}
+				else if(cJSON_IsArray(json_address) && cJSON_GetArraySize(json_address) > 0)
+					row.items = json_address;
+				else
+				{
+					return send_json_error(api, 400,
+					                       "bad_request",
+					                       "Invalid request: No valid \"address\" in payload (must be either string or array)",
+					                       NULL);
+				}
+				break;
+			}
+
+			// Aggregate types (and gravity) are not handled by this routine
+			case GRAVITY_DOMAINLIST_ALL_ALL:
+			case GRAVITY_DOMAINLIST_ALL_EXACT:
+			case GRAVITY_DOMAINLIST_ALL_REGEX:
+			case GRAVITY_DOMAINLIST_ALLOW_ALL:
+			case GRAVITY_DOMAINLIST_DENY_ALL:
+			case GRAVITY_GRAVITY:
+			case GRAVITY_ANTIGRAVITY:
+				return send_json_error(api, 400, // 400 Bad Request
+				                       "bad_request",
+				                       "Aggregate types (and gravity) are not handled by this routine",
+				                       NULL);
+		}
+	}
+	else
+	{
+		// PUT = Use URI item
+		row.items = cJSON_CreateArray();
+		cJSON_AddItemToArray(row.items, cJSON_CreateStringReference(item));
+		allocated_json = true;
+	}
+
+	cJSON *json_comment = cJSON_GetObjectItemCaseSensitive(api->payload.json, "comment");
+	if(cJSON_IsString(json_comment) && strlen(json_comment->valuestring) > 0)
+		row.comment = json_comment->valuestring;
+	else
+		row.comment = NULL; // Default value
+
+
+	// Check if there is a type field in the payload (only for lists)
+	if(listtype == GRAVITY_ADLISTS)
+	{
+		cJSON *json_type = cJSON_GetObjectItemCaseSensitive(api->payload.json, "type");
+		if(cJSON_IsString(json_type) && strlen(json_type->valuestring) > 0)
+			row.type_int = strcasecmp(json_type->valuestring, "allow") == 0 ? ADLIST_ALLOW : ADLIST_BLOCK;
+		else
+		{
+			return send_json_error(api, 400,
+			                       "bad_request",
+			                       "Invalid request: No valid item \"type\" in payload",
+			                       NULL);
+		}
+	}
+	else if(listtype == GRAVITY_ADLISTS_BLOCK)
+		row.type_int = ADLIST_BLOCK;
+	else if(listtype == GRAVITY_ADLISTS_ALLOW)
+		row.type_int = ADLIST_ALLOW;
+	else
+	{
+		cJSON *json_type = cJSON_GetObjectItemCaseSensitive(api->payload.json, "type");
+		if(cJSON_IsString(json_type) && strlen(json_type->valuestring) > 0)
+			row.type = json_type->valuestring;
+		else
+			row.type = NULL; // Default value
+	}
+
+	cJSON *json_kind = cJSON_GetObjectItemCaseSensitive(api->payload.json, "kind");
+	if(cJSON_IsString(json_kind) && strlen(json_kind->valuestring) > 0)
+		row.kind = json_kind->valuestring;
+	else
+		row.kind = NULL; // Default value
+
+	cJSON *json_enabled = cJSON_GetObjectItemCaseSensitive(api->payload.json, "enabled");
+	if (cJSON_IsBool(json_enabled))
+		row.enabled = cJSON_IsTrue(json_enabled);
+	else
+		row.enabled = true; // Default value
+
+	cJSON *json_name = cJSON_GetObjectItemCaseSensitive(api->payload.json, "name");
+	if(cJSON_IsString(json_name) && strlen(json_name->valuestring) > 0)
+		row.name = json_name->valuestring;
+	else
+		row.name = NULL; // Default value
+
+	bool okay = true;
+	char *regex_msg = NULL;
+	if(listtype == GRAVITY_DOMAINLIST_ALLOW_REGEX || listtype == GRAVITY_DOMAINLIST_DENY_REGEX)
+	{
+		// Test validity of this regex
+		regexData regex = { 0 };
+		cJSON *it = NULL;
+		cJSON_ArrayForEach(it, row.items)
+		{
+			// If any element isn't a string, break early
+			if(!cJSON_IsString(it))
+			{
+				okay = false;
+				break;
+			}
+
+			// Check every array element for its validity
+			okay = compile_regex(it->valuestring, &regex, &regex_msg);
+
+			// Free regex after successful compilation
+			if(regex.available)
+			{
+				regfree(&regex.regex);
+				free(regex.string);
+			}
+
+			// Fail fast if any regex in the passed array is invalid
+			if(!okay)
+				break;
+		}
+	}
+	else if(!spaces_allowed)
+	{
+		cJSON *it = NULL;
+		cJSON_ArrayForEach(it, row.items)
+		{
+			// If any element isn't a string, break early
+			if(!cJSON_IsString(it))
+			{
+				okay = false;
+				break;
+			}
+
+			// Check validity: Spaces are not allowed in any domain/URL
+			if(strchr(it->valuestring, ' ') != NULL ||
+			   strchr(it->valuestring, '\t') != NULL ||
+			   strchr(it->valuestring, '\n') != NULL)
+			{
+				if(allocated_json)
+					cJSON_free(row.items);
+				return send_json_error(api, 400, // 400 Bad Request
+				                       "bad_request",
+				                       "Spaces, newlines and tabs are not allowed in domains and URLs",
+				                       it->valuestring);
+			}
+
+			if(listtype == GRAVITY_DOMAINLIST_ALLOW_EXACT ||
+			   listtype == GRAVITY_DOMAINLIST_DENY_EXACT)
+			{
+				char *punycode = NULL;
+				const int rc = idn2_to_ascii_lz(it->valuestring, &punycode, IDN2_NFC_INPUT | IDN2_NONTRANSITIONAL);
+				if (rc != IDN2_OK)
+				{
+					// Invalid domain name
+					return send_json_error(api, 400,
+					                       "bad_request",
+					                       "Invalid request: Invalid domain name",
+					                       idn2_strerror(rc));
+				}
+				// Convert punycode domain to lowercase
+				for(unsigned int i = 0u; i < strlen(punycode); i++)
+					punycode[i] = tolower(punycode[i]);
+
+				// Validate punycode domain
+				// This will reject domains like äöü{{{.com
+				// which convert to xn--{{{-pla4gpb.com
+				if(!valid_domain(punycode, strlen(punycode), false))
+				{
+					if(allocated_json)
+						cJSON_free(row.items);
+					return send_json_error(api, 400, // 400 Bad Request
+							"bad_request",
+							"Invalid domain",
+							it->valuestring);
+				}
+
+				// Replace domain with punycode version
+				if(!(it->type & cJSON_IsReference))
+					free(it->valuestring);
+				it->valuestring = punycode;
+				// Remove reference flag
+				it->type &= ~cJSON_IsReference;
+			}
+		}
+	}
+
+	// Fail fast if any regex in the passed array is invalid
+	if(!okay)
+	{
+		// Send error reply
+		if(allocated_json)
+			cJSON_free(row.items);
+		return send_json_error_free(api, 400, // 400 Bad Request
+		                            "regex_error",
+		                            "Regex validation failed",
+		                            regex_msg, true);
+	}
+
+	// Try to add item(s) to table
+	const char *sql_msg = NULL;
+	cJSON *elem = NULL;
+	cJSON *processed = JSON_NEW_OBJECT();
+	cJSON *errors = JSON_NEW_ARRAY();
+	cJSON *success = JSON_NEW_ARRAY();
+	cJSON_AddItemToObject(processed, "errors", errors);
+	cJSON_AddItemToObject(processed, "success", success);
+	cJSON_ArrayForEach(elem, row.items)
+	{
+		row.item = elem->valuestring;
+		if((okay = gravityDB_addToTable(listtype, &row, &sql_msg, api->method)))
+		{
+			if(listtype != GRAVITY_GROUPS)
+			{
+				cJSON *groups = cJSON_GetObjectItemCaseSensitive(api->payload.json, "groups");
+				if(groups != NULL)
+					okay = gravityDB_edit_groups(listtype, groups, &row, &sql_msg);
+				else
+					// The groups array is optional, we still succeed if it
+					// is omitted (groups stay as they are)
+					okay = true;
+			}
+			else
+			{
+				// Groups cannot be assigned to groups
+				okay = true;
+			}
+		}
+
+		cJSON *details = JSON_NEW_OBJECT();
+		JSON_COPY_STR_TO_OBJECT(details, "item", row.item);
+		if(!okay)
+			JSON_COPY_STR_TO_OBJECT(details, "error", sql_msg);
+		cJSON_AddItemToArray(okay ? success : errors, details);
+	}
+
+	// Inform the resolver that it needs to reload gravity
+	set_event(RELOAD_GRAVITY);
+
+	int response_code = 201; // 201 - Created
+	if(api->method == HTTP_PUT)
+		response_code = 200; // 200 - OK
+
+	// Add "Location" header to response
+	if(snprintf(pi_hole_extra_headers, sizeof(pi_hole_extra_headers), "Location: %s/%s", api->action_path, row.item) >= (int)sizeof(pi_hole_extra_headers))
+	{
+		// This may happen for *extremely* long URLs but is not issue in
+		// itself. Merely add a warning to the log file
+		log_warn("Could not add Location header to response: URL too long");
+
+		// Truncate location by replacing the last characters with "...\0"
+		pi_hole_extra_headers[sizeof(pi_hole_extra_headers)-4] = '.';
+		pi_hole_extra_headers[sizeof(pi_hole_extra_headers)-3] = '.';
+		pi_hole_extra_headers[sizeof(pi_hole_extra_headers)-2] = '.';
+		pi_hole_extra_headers[sizeof(pi_hole_extra_headers)-1] = '\0';
+	}
+
+	// Send GET style reply
+	const int ret = api_list_read(api, response_code, listtype, row.item, processed);
+
+	// Free allocated memory
+	if(allocated_json)
+		cJSON_free(row.items);
+
+	return ret;
+}
+
+static int api_list_remove(struct ftl_conn *api,
+                           const enum gravity_list_type listtype,
+                           const char *item)
+{
+	const char *sql_msg = NULL;
+	cJSON *array = api->payload.json;
+	bool allocated_json = false;
+
+	// If this is not a :batchDelete call, then the item is specified in the
+	// URI, not in the payload. Create a JSON array with the item and use
+	// that instead
+	const bool isBatchDelete = api->opts.flags & API_BATCHDELETE;
+
+	// If this is a domain callback, we need to translate type/kind into an
+	// integer for use in the database
+	if(listtype == GRAVITY_DOMAINLIST_ALLOW_EXACT ||
+	   listtype == GRAVITY_DOMAINLIST_DENY_EXACT ||
+	   listtype == GRAVITY_DOMAINLIST_ALLOW_REGEX ||
+	   listtype == GRAVITY_DOMAINLIST_DENY_REGEX ||
+	   listtype == GRAVITY_ADLISTS_BLOCK ||
+	   listtype == GRAVITY_ADLISTS_ALLOW)
+	{
+		int type = -1;
+		switch (listtype)
+		{
+			case GRAVITY_DOMAINLIST_ALLOW_EXACT:
+				type = 0;
+				break;
+			case GRAVITY_DOMAINLIST_DENY_EXACT:
+				type = 1;
+				break;
+			case GRAVITY_DOMAINLIST_ALLOW_REGEX:
+				type = 2;
+				break;
+			case GRAVITY_DOMAINLIST_DENY_REGEX:
+				type = 3;
+				break;
+			case GRAVITY_ADLISTS_BLOCK:
+				type = ADLIST_BLOCK;
+				break;
+			case GRAVITY_ADLISTS_ALLOW:
+				type = ADLIST_ALLOW;
+				break;
+			// Not handled herein
+			case GRAVITY_GROUPS:
+			case GRAVITY_ADLISTS:
+			case GRAVITY_CLIENTS:
+			case GRAVITY_GRAVITY:
+			case GRAVITY_ANTIGRAVITY:
+			case GRAVITY_DOMAINLIST_ALLOW_ALL:
+			case GRAVITY_DOMAINLIST_DENY_ALL:
+			case GRAVITY_DOMAINLIST_ALL_EXACT:
+			case GRAVITY_DOMAINLIST_ALL_REGEX:
+			case GRAVITY_DOMAINLIST_ALL_ALL:
+			default:
+				break;
+		}
+
+		// Create new JSON array with the item and type:
+		// array = [{"item": "example.com", "type": 0}]
+		array = cJSON_CreateArray();
+		cJSON *obj = cJSON_CreateObject();
+		cJSON_AddItemToObject(obj, "item", cJSON_CreateStringReference(item));
+		cJSON_AddItemToObject(obj, "type", cJSON_CreateNumber(type));
+		cJSON_AddItemToArray(array, obj);
+		allocated_json = true;
+	}
+	else if(isBatchDelete && listtype == GRAVITY_DOMAINLIST_ALL_ALL)
+	{
+		// Loop over all items and parse type/kind for each item
+		cJSON *it = NULL;
+		cJSON_ArrayForEach(it, array)
+		{
+			if(!cJSON_IsObject(it))
+			{
+				return send_json_error(api, 400,
+				                       "bad_request",
+				                       "Invalid request: Batch delete requires an array of objects",
+				                       NULL);
+			}
+
+			// Check if item is a string
+			cJSON *json_item = cJSON_GetObjectItemCaseSensitive(it, "item");
+			if(!cJSON_IsString(json_item))
+			{
+				return send_json_error(api, 400,
+				                       "bad_request",
+				                       "Invalid request: Batch delete requires an array of objects with \"item\" as string",
+				                       NULL);
+			}
+
+			// Check if type and kind are both present and strings
+			cJSON *json_type = cJSON_GetObjectItemCaseSensitive(it, "type");
+			cJSON *json_kind = cJSON_GetObjectItemCaseSensitive(it, "kind");
+			if(!cJSON_IsString(json_type) || !cJSON_IsString(json_kind))
+			{
+				return send_json_error(api, 400,
+				                       "bad_request",
+				                       "Invalid request: Batch delete requires an array of objects with \"type\" and \"kind\" as string",
+				                       NULL);
+			}
+
+			// Parse type and kind
+			// 0 = allow exact
+			// 1 = deny exact
+			// 2 = allow regex
+			// 3 = deny regex
+			int type = -1;
+			if(strcasecmp(json_type->valuestring, "allow") == 0)
+			{
+				if(strcasecmp(json_kind->valuestring, "exact") == 0)
+					type = 0;
+				else if(strcasecmp(json_kind->valuestring, "regex") == 0)
+					type = 2;
+			}
+			else if(strcasecmp(json_type->valuestring, "deny") == 0)
+			{
+				if(strcasecmp(json_kind->valuestring, "exact") == 0)
+					type = 1;
+				else if(strcasecmp(json_kind->valuestring, "regex") == 0)
+					type = 3;
+			}
+
+			// Check if type/kind combination is valid
+			if(type == -1)
+			{
+				return send_json_error(api, 400,
+				                       "bad_request",
+				                       "Invalid request: Batch delete requires an valid combination of \"type\" and \"kind\" for each object",
+				                       NULL);
+			}
+
+			// Replace type/kind with integer type
+			// array = [{"item": "example.com", "type": 0}]
+			cJSON_DeleteItemFromObject(it, "type");
+			cJSON_DeleteItemFromObject(it, "kind");
+			cJSON_AddNumberToObject(it, "type", type);
+		}
+	}
+	else if(!isBatchDelete)
+	{
+		// Create array with object (used for clients, groups, lists)
+		// array = [{"item": <item>}]
+		array = cJSON_CreateArray();
+		cJSON *obj = cJSON_CreateObject();
+		cJSON_AddItemToObject(obj, "item", cJSON_CreateStringReference(item));
+		cJSON_AddItemToArray(array, obj);
+		allocated_json = true;
+	}
+
+	// Verify that the payload is an array of objects each containing an
+	// item
+	if(isBatchDelete)
+	{
+		cJSON *it = NULL;
+		cJSON_ArrayForEach(it, array)
+		{
+			if(!cJSON_IsObject(it))
+			{
+				return send_json_error(api, 400,
+				                       "bad_request",
+				                       "Invalid request: Batch delete requires an array of objects",
+				                       NULL);
+			}
+
+			// Check if item is a string
+			cJSON *json_item = cJSON_GetObjectItemCaseSensitive(it, "item");
+			if(!cJSON_IsString(json_item))
+			{
+				return send_json_error(api, 400,
+				                       "bad_request",
+				                       "Invalid request: Batch delete requires an array of objects with \"item\" as string",
+				                       NULL);
+			}
+		}
+	}
+
+	// From here on, we can assume the JSON payload is valid
+	unsigned int deleted = 0u;
+	if(gravityDB_delFromTable(listtype, array, &deleted, &sql_msg))
+	{
+		// Inform the resolver that it needs to reload gravity
+		set_event(RELOAD_GRAVITY);
+
+		// Free memory allocated above
+		if(allocated_json)
+			cJSON_free(array);
+
+		// Send empty reply with codes:
+		// - 204 No Content (if any items were deleted)
+		// - 404 Not Found (if no items were deleted)
+		cJSON *json = JSON_NEW_OBJECT();
+		JSON_SEND_OBJECT_CODE(json, deleted > 0u ? 204 : 404);
+	}
+	else
+	{
+		// Free memory allocated above
+		if(allocated_json)
+			cJSON_free(array);
+
+		// Send error reply
+		return send_json_error(api, 400,
+		                       "database_error",
+		                       "Could not remove entries from table",
+		                       sql_msg);
+	}
+}
+
+int api_list(struct ftl_conn *api)
+{
+	enum gravity_list_type listtype;
+	bool can_modify = false;
+	bool batchDelete = false;
+	if((api->item = startsWith("/api/groups", api)) != NULL)
+	{
+		listtype = GRAVITY_GROUPS;
+		can_modify = true;
+	}
+	else if((api->item = startsWith("/api/groups:batchDelete", api)) != NULL)
+	{
+		listtype = GRAVITY_GROUPS;
+		can_modify = true;
+		batchDelete = true;
+	}
+	else if((api->item = startsWith("/api/lists", api)) != NULL)
+	{
+		listtype = GRAVITY_ADLISTS;
+		can_modify = true;
+	}
+	else if((api->item = startsWith("/api/lists:batchDelete", api)) != NULL)
+	{
+		listtype = GRAVITY_ADLISTS;
+		can_modify = true;
+		batchDelete = true;
+	}
+	else if((api->item = startsWith("/api/clients", api)) != NULL)
+	{
+		listtype = GRAVITY_CLIENTS;
+		can_modify = true;
+	}
+	else if((api->item = startsWith("/api/clients:batchDelete", api)) != NULL)
+	{
+		listtype = GRAVITY_CLIENTS;
+		can_modify = true;
+		batchDelete = true;
+	}
+	else if((api->item = startsWith("/api/domains/allow/exact", api)) != NULL)
+	{
+		listtype = GRAVITY_DOMAINLIST_ALLOW_EXACT;
+		can_modify = true;
+	}
+	else if((api->item = startsWith("/api/domains/allow/regex", api)) != NULL)
+	{
+		listtype = GRAVITY_DOMAINLIST_ALLOW_REGEX;
+		can_modify = true;
+	}
+	else if((api->item = startsWith("/api/domains/allow", api)) != NULL)
+	{
+		listtype = GRAVITY_DOMAINLIST_ALLOW_ALL;
+	}
+	else if((api->item = startsWith("/api/domains/deny/exact", api)) != NULL)
+	{
+		listtype = GRAVITY_DOMAINLIST_DENY_EXACT;
+		can_modify = true;
+	}
+	else if((api->item = startsWith("/api/domains/deny/regex", api)) != NULL)
+	{
+		listtype = GRAVITY_DOMAINLIST_DENY_REGEX;
+		can_modify = true;
+	}
+	else if((api->item = startsWith("/api/domains/deny", api)) != NULL)
+	{
+		listtype = GRAVITY_DOMAINLIST_DENY_ALL;
+	}
+	else if((api->item = startsWith("/api/domains/exact", api)) != NULL)
+	{
+		listtype = GRAVITY_DOMAINLIST_ALL_EXACT;
+	}
+	else if((api->item = startsWith("/api/domains/regex", api)) != NULL)
+	{
+		listtype = GRAVITY_DOMAINLIST_ALL_REGEX;
+	}
+	else if((api->item = startsWith("/api/domains", api)) != NULL)
+	{
+		listtype = GRAVITY_DOMAINLIST_ALL_ALL;
+	}
+	else if((api->item = startsWith("/api/domains:batchDelete", api)) != NULL)
+	{
+		listtype = GRAVITY_DOMAINLIST_ALL_ALL;
+		can_modify = true;
+		batchDelete = true;
+	}
+	else
+	{
+			return send_json_error(api, 400,
+			                       "bad_request",
+			                       "Invalid request: Specified endpoint not available",
+			                       api->request->local_uri_raw);
+	}
+
+	// If this is a request for a list, we check if there is a request
+	// parameter narrowing down which kind of list. If so, we modify the
+	// list type accordingly
+	if(listtype == GRAVITY_ADLISTS && api->request->query_string != NULL)
+	{
+		// Check if there is a type parameter
+		char typestr[16] = { 0 };
+		if(get_string_var(api->request->query_string, "type", typestr, sizeof(typestr)) > 0)
+		{
+			if(strcasecmp(typestr, "allow") == 0)
+				listtype = GRAVITY_ADLISTS_ALLOW;
+			else if(strcasecmp(typestr, "block") == 0)
+				listtype = GRAVITY_ADLISTS_BLOCK;
+			else
+			{
+				// Invalid type parameter
+				return send_json_error(api, 400,
+				                       "bad_request",
+				                       "Invalid request: Invalid type parameter (should be either \"allow\" or \"block\")",
+				                       api->request->query_string);
+			}
+		}
+	}
+
+	if(api->method == HTTP_GET)
+	{
+		// Read list item identified by URI (or read them all)
+		// We would not actually need the SHM lock here, however, we do
+		// this for simplicity to ensure nobody else is editing the
+		// lists while we're doing this here
+		lock_shm();
+		const int ret = api_list_read(api, 200, listtype, api->item, NULL);
+		unlock_shm();
+		return ret;
+	}
+	else if(can_modify && api->method == HTTP_PUT)
+	{
+		// Add/update item identified by URI
+		if(api->item != NULL && strlen(api->item) == 0)
+		{
+			return send_json_error(api, 400,
+			                       "uri_error",
+			                       "Invalid request: Specify item in URI",
+			                       NULL);
+		}
+		else
+		{
+			// We would not actually need the SHM lock here,
+			// however, we do this for simplicity to ensure nobody
+			// else is editing the lists while we're doing this here
+			lock_shm();
+			const int ret = api_list_write(api, listtype, api->item);
+			unlock_shm();
+			return ret;
+		}
+	}
+	else if(can_modify && api->method == HTTP_POST && !batchDelete)
+	{
+		// Add item to list identified by payload
+		if(api->item != NULL && strlen(api->item) != 0)
+		{
+			return send_json_error(api, 400,
+			                       "uri_error",
+			                       "Invalid request: Specify item in payload, not as URI parameter",
+			                       api->item);
+		}
+		else
+		{
+			// We would not actually need the SHM lock here,
+			// however, we do this for simplicity to ensure nobody
+			// else is editing the lists while we're doing this here
+			lock_shm();
+			const int ret = api_list_write(api, listtype, api->item);
+			unlock_shm();
+			return ret;
+		}
+	}
+	else if(can_modify && (api->method == HTTP_DELETE || (api->method == HTTP_POST && batchDelete)))
+	{
+		// Delete item from list
+		// We would not actually need the SHM lock here, however, we do
+		// this for simplicity to ensure nobody else is editing the
+		// lists while we're doing this here
+		lock_shm();
+		const int ret = api_list_remove(api, listtype, api->item);
+		unlock_shm();
+		return ret;
+	}
+	else if(!can_modify)
+	{
+		// This list type cannot be modified (e.g., ALL_ALL)
+		return send_json_error(api, 400,
+		                       "uri_error",
+		                       "Invalid request: Specify list to modify more precisely",
+		                       api->request->local_uri_raw);
+	}
+	else
+	{
+		// This results in error 404
+		return 0;
+	}
+}

src/api/logs.c

@@ -0,0 +1,99 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2023 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  API Implementation /api/logs
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#include "FTL.h"
+#include "webserver/http-common.h"
+#include "webserver/json_macros.h"
+#include "api/api.h"
+// struct fifologData
+#include "log.h"
+#include "config/config.h"
+// main_pid()
+#include "signals.h"
+
+// fifologData is allocated in shared memory for cross-fork compatibility
+int api_logs(struct ftl_conn *api)
+{
+	unsigned int start = 0u;
+	if(api->request->query_string != NULL)
+	{
+		// Does the user request an ID to sent from?
+		unsigned int nextID;
+		if(get_uint_var(api->request->query_string, "nextID", &nextID))
+		{
+			if(nextID >= fifo_log->logs[api->opts.which].next_id)
+			{
+				// Do not return any data
+				start = LOG_SIZE;
+			}
+			else if((fifo_log->logs[api->opts.which].next_id > LOG_SIZE) && nextID < (fifo_log->logs[api->opts.which].next_id) - LOG_SIZE)
+			{
+				// Requested an ID smaller than the lowest one we have
+				// We return the entire buffer
+				start = 0u;
+			}
+			else if(fifo_log->logs[api->opts.which].next_id >= LOG_SIZE)
+			{
+				// Reply with partial buffer, measure from the end
+				// (the log is full)
+				start = LOG_SIZE - (fifo_log->logs[api->opts.which].next_id - nextID);
+			}
+			else
+			{
+				// Reply with partial buffer, measure from the start
+				// (the log is not yet full)
+				start = nextID;
+			}
+		}
+	}
+
+	// Process data
+	cJSON *json = JSON_NEW_OBJECT();
+	cJSON *log = JSON_NEW_ARRAY();
+	for(unsigned int i = start; i < LOG_SIZE; i++)
+	{
+		if(fifo_log->logs[api->opts.which].timestamp[i] < 1.0)
+		{
+			// Uninitialized buffer entry
+			break;
+		}
+
+		cJSON *entry = JSON_NEW_OBJECT();
+		JSON_ADD_NUMBER_TO_OBJECT(entry, "timestamp", fifo_log->logs[api->opts.which].timestamp[i]);
+		JSON_REF_STR_IN_OBJECT(entry, "message", fifo_log->logs[api->opts.which].message[i]);
+		JSON_REF_STR_IN_OBJECT(entry, "prio", fifo_log->logs[api->opts.which].prio[i]);
+		JSON_ADD_ITEM_TO_ARRAY(log, entry);
+	}
+	JSON_ADD_ITEM_TO_OBJECT(json, "log", log);
+	JSON_ADD_NUMBER_TO_OBJECT(json, "nextID", fifo_log->logs[api->opts.which].next_id);
+	JSON_ADD_NUMBER_TO_OBJECT(json, "pid", main_pid());
+
+	// Add file name
+	const char *logfile = NULL;
+	switch(api->opts.which)
+	{
+		case FIFO_FTL:
+			logfile = config.files.log.ftl.v.s;
+			break;
+		case FIFO_DNSMASQ:
+			logfile = config.files.log.dnsmasq.v.s;
+			break;
+		case FIFO_WEBSERVER:
+			logfile = config.files.log.webserver.v.s;
+			break;
+		case FIFO_MAX:
+			// This should never happen
+			break;
+	}
+	JSON_REF_STR_IN_OBJECT(json, "file", logfile);
+
+	// Send data
+	JSON_SEND_OBJECT(json);
+}

src/api/msgpack.c

@@ -1,120 +0,0 @@
-/* Pi-hole: A black hole for Internet advertisements
-*  (c) 2017 Pi-hole, LLC (https://pi-hole.net)
-*  Network-wide ad blocking via your own hardware.
-*
-*  FTL Engine
-*  MessagePack serialization
-*
-*  This file is copyright under the latest version of the EUPL.
-*  Please see LICENSE file for your rights under this license. */
-
-#include "FTL.h"
-#include "api.h"
-#include "socket.h"
-#include "log.h"
-
-void pack_eom(const int sock) {
-	// This byte is explicitly never used in the MessagePack spec, so it is perfect to use as an EOM for this API.
-	uint8_t eom = 0xc1;
-	write(sock, &eom, sizeof(eom));
-}
-
-static void pack_basic(const int sock, const uint8_t format, const void *value, const size_t size) {
-	write(sock, &format, sizeof(format));
-	write(sock, value, size);
-}
-
-static uint64_t __attribute__((const)) leToBe64(const uint64_t value) {
-	const char *ptr = (char *) &value;
-	uint32_t part1, part2;
-
-	// Copy the two halves of the 64 bit input into uint32_t's so we can use htonl
-	memcpy(&part1, ptr, 4);
-	memcpy(&part2, ptr + 4, 4);
-
-	// Flip each half around
-	part1 = htonl(part1);
-	part2 = htonl(part2);
-
-	// Arrange them to form the big-endian version of the original input
-	return (uint64_t) part1 << 32 | part2;
-}
-
-void pack_bool(const int sock, const bool value) {
-	uint8_t packed = (uint8_t) (value ? 0xc3 : 0xc2);
-	write(sock, &packed, sizeof(packed));
-}
-
-void pack_uint8(const int sock, const uint8_t value) {
-	pack_basic(sock, 0xcc, &value, sizeof(value));
-}
-
-void pack_uint64(const int sock, const uint64_t value) {
-	const uint64_t bigEValue = leToBe64(value);
-	pack_basic(sock, 0xcf, &bigEValue, sizeof(bigEValue));
-}
-
-void pack_int32(const int sock, const int32_t value) {
-	const uint32_t bigEValue = htonl((uint32_t) value);
-	pack_basic(sock, 0xd2, &bigEValue, sizeof(bigEValue));
-}
-
-void pack_int64(const int sock, const int64_t value) {
-	// Need to use memcpy to do a direct copy without reinterpreting the bytes (making negatives into positives).
-	// It should get optimized away.
-	uint64_t bigEValue;
-	memcpy(&bigEValue, &value, sizeof(bigEValue));
-	bigEValue = leToBe64(bigEValue);
-	pack_basic(sock, 0xd3, &bigEValue, sizeof(bigEValue));
-}
-
-void pack_float(const int sock, const float value) {
-	// Need to use memcpy to do a direct copy without reinterpreting the bytes. It should get optimized away.
-	uint32_t bigEValue;
-	memcpy(&bigEValue, &value, sizeof(bigEValue));
-	bigEValue = htonl(bigEValue);
-	pack_basic(sock, 0xca, &bigEValue, sizeof(bigEValue));
-}
-
-// Return true if successful
-bool pack_fixstr(const int sock, const char *string) {
-	// Make sure that the length is less than 32
-	const size_t length = strlen(string);
-
-	if(length >= 32) {
-		logg("Tried to send a fixstr longer than 31 bytes!");
-		return false;
-	}
-
-	const uint8_t format = (uint8_t) (0xA0 | length);
-	write(sock, &format, sizeof(format));
-	write(sock, string, length);
-
-	return true;
-}
-
-// Return true if successful
-bool pack_str32(const int sock, const char *string) {
-	// Make sure that the length is less than 4294967296
-	const size_t length = strlen(string);
-
-	if(length >= 2147483648u) {
-		logg("Tried to send a str32 longer than 2147483647 bytes!");
-		return false;
-	}
-
-	const uint8_t format = 0xdb;
-	write(sock, &format, sizeof(format));
-	const uint32_t bigELength = htonl((uint32_t) length);
-	write(sock, &bigELength, sizeof(bigELength));
-	write(sock, string, length);
-
-	return true;
-}
-
-void pack_map16_start(const int sock, const uint16_t length) {
-	const uint8_t format = 0xde;
-	write(sock, &format, sizeof(format));
-	const uint16_t bigELength = htons(length);
-	write(sock, &bigELength, sizeof(bigELength));
-}

src/api/network.c

@@ -0,0 +1,582 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2019 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  API Implementation /api/network
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#include "FTL.h"
+#include "webserver/http-common.h"
+#include "webserver/json_macros.h"
+#include "api/api.h"
+// Routing information and flags
+#include <net/route.h>
+// Iterate through directories
+#include <dirent.h>
+// networkrecord
+#include "database/network-table.h"
+// dbopen(false, )
+#include "database/common.h"
+// attach_database()
+#include "database/query-table.h"
+// config struct
+#include "config/config.h"
+
+static bool getDefaultInterface(char iface[IF_NAMESIZE], in_addr_t *gw)
+{
+	// Get IPv4 default route gateway and associated interface
+	unsigned long dest_r = 0, gw_r = 0;
+	unsigned int flags = 0u;
+	int metric = 0, minmetric = __INT_MAX__;
+
+	FILE *file;
+	if((file = fopen("/proc/net/route", "r")))
+	{
+		// Parse /proc/net/route - the kernel's IPv4 routing table
+		char buf[1024] = { 0 };
+		while(fgets(buf, sizeof(buf), file))
+		{
+			char iface_r[IF_NAMESIZE] = { 0 };
+			if(sscanf(buf, "%15s %lx %lx %x %*i %*i %i", iface_r, &dest_r, &gw_r, &flags, &metric) != 5)
+				continue;
+
+			// Only analyze routes which are UP and whose
+			// destinations are a gateway
+			if(!(flags & RTF_UP) || !(flags & RTF_GATEWAY))
+				continue;
+
+			// Only analyze "catch all" routes (destination 0.0.0.0)
+			if(dest_r != 0)
+				continue;
+
+			// Store default gateway, overwrite if we find a route with
+			// a lower metric
+			if(metric < minmetric)
+			{
+				minmetric = metric;
+				*gw = gw_r;
+				strcpy(iface, iface_r);
+
+				log_debug(DEBUG_API, "Reading interfaces: flags: %u, addr: %s, iface: %s, metric: %i, minmetric: %i",
+				          flags, inet_ntoa(*(struct in_addr *) gw), iface, metric, minmetric);
+			}
+		}
+		fclose(file);
+	}
+	else
+		log_err("Cannot read /proc/net/route: %s", strerror(errno));
+
+	// Return success based on having found the default gateway's address
+	return gw != 0;
+}
+
+int api_network_gateway(struct ftl_conn *api)
+{
+	in_addr_t gw = 0;
+	char iface[IF_NAMESIZE] = { 0 };
+
+	// Get default interface
+	getDefaultInterface(iface, &gw);
+
+	// Generate JSON response
+	cJSON *json = JSON_NEW_OBJECT();
+	const char *gwaddr = inet_ntoa(*(struct in_addr *) &gw);
+	JSON_COPY_STR_TO_OBJECT(json, "address", gwaddr);
+	JSON_REF_STR_IN_OBJECT(json, "interface", iface);
+	JSON_SEND_OBJECT(json);
+}
+
+int api_network_interfaces(struct ftl_conn *api)
+{
+	cJSON *json = JSON_NEW_OBJECT();
+
+	// Get interface with default route
+	in_addr_t gw = 0;
+	char default_iface[IF_NAMESIZE] = { 0 };
+	getDefaultInterface(default_iface, &gw);
+
+	// Enumerate and list interfaces
+	// Loop over interfaces and extract information
+	DIR *dfd;
+	FILE *f;
+	struct dirent *dp;
+	size_t tx_sum = 0, rx_sum = 0;
+	char fname[64 + IF_NAMESIZE] = { 0 };
+	char readbuffer[1024] = { 0 };
+
+	// Open /sys/class/net directory
+	if ((dfd = opendir("/sys/class/net")) == NULL)
+	{
+		log_err("API: Cannot access /sys/class/net");
+		return 500;
+	}
+
+	// Get IP addresses of all interfaces on this machine
+	struct ifaddrs *ifap = NULL;
+	if(getifaddrs(&ifap) == -1)
+		log_err("API: Cannot get interface addresses: %s", strerror(errno));
+
+	cJSON *interfaces = JSON_NEW_ARRAY();
+	// Walk /sys/class/net directory
+	while ((dp = readdir(dfd)) != NULL)
+	{
+		// Skip "." and ".."
+		if(strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0)
+			continue;
+
+		// Create new interface record
+		cJSON *iface = JSON_NEW_OBJECT();
+
+		// Extract interface name
+		const char *iface_name = dp->d_name;
+		JSON_COPY_STR_TO_OBJECT(iface, "name", iface_name);
+
+		// Is this the default interface?
+		const bool is_default_iface = strcmp(iface_name, default_iface) == 0;
+		JSON_ADD_BOOL_TO_OBJECT(iface, "default", is_default_iface);
+
+		// Extract carrier status
+		bool carrier = false;
+		snprintf(fname, sizeof(fname)-1, "/sys/class/net/%s/carrier", iface_name);
+		if((f = fopen(fname, "r")) != NULL)
+		{
+			if(fgets(readbuffer, sizeof(readbuffer)-1, f) != NULL)
+				carrier = readbuffer[0] == '1';
+			fclose(f);
+		}
+		else
+			log_err("Cannot read %s: %s", fname, strerror(errno));
+		JSON_ADD_BOOL_TO_OBJECT(iface, "carrier", carrier);
+
+		// Extract link speed (may not be possible, e.g., for WiFi devices with dynamic link speeds)
+		int speed = -1;
+		snprintf(fname, sizeof(fname)-1, "/sys/class/net/%s/speed", iface_name);
+		if((f = fopen(fname, "r")) != NULL)
+		{
+			if(fscanf(f, "%i", &(speed)) != 1)
+				speed = -1;
+			fclose(f);
+		}
+		else
+			log_err("Cannot read %s: %s", fname, strerror(errno));
+		JSON_ADD_NUMBER_TO_OBJECT(iface, "speed", speed);
+
+		// Get total transmitted bytes
+		ssize_t tx_bytes = -1;
+		snprintf(fname, sizeof(fname)-1, "/sys/class/net/%s/statistics/tx_bytes", iface_name);
+		if((f = fopen(fname, "r")) != NULL)
+		{
+			if(fscanf(f, "%zi", &(tx_bytes)) != 1)
+				tx_bytes = -1;
+			fclose(f);
+		}
+		else
+			log_err("Cannot read %s: %s", fname, strerror(errno));
+
+		// Format transmitted bytes
+		double tx = 0.0;
+		char tx_unit[3] = { 0 };
+		format_memory_size(tx_unit, tx_bytes, &tx);
+		if(tx_unit[0] != '\0')
+			tx_unit[1] = 'B';
+
+		// Add transmitted bytes to interface record
+		cJSON *tx_json = JSON_NEW_OBJECT();
+		JSON_ADD_NUMBER_TO_OBJECT(tx_json, "num", tx);
+		JSON_COPY_STR_TO_OBJECT(tx_json, "unit", tx_unit);
+		JSON_ADD_ITEM_TO_OBJECT(iface, "tx", tx_json);
+
+		// Get total received bytes
+		ssize_t rx_bytes = -1;
+		snprintf(fname, sizeof(fname)-1, "/sys/class/net/%s/statistics/rx_bytes", iface_name);
+		if((f = fopen(fname, "r")) != NULL)
+		{
+			if(fscanf(f, "%zi", &(rx_bytes)) != 1)
+				rx_bytes = -1;
+			fclose(f);
+		}
+		else
+			log_err("Cannot read %s: %s", fname, strerror(errno));
+
+		// Format received bytes
+		double rx = 0.0;
+		char rx_unit[3] = { 0 };
+		format_memory_size(rx_unit, rx_bytes, &rx);
+		if(rx_unit[0] != '\0')
+			rx_unit[1] = 'B';
+
+		// Add received bytes to JSON object
+		cJSON *rx_json = JSON_NEW_OBJECT();
+		JSON_ADD_NUMBER_TO_OBJECT(rx_json, "num", rx);
+		JSON_COPY_STR_TO_OBJECT(rx_json, "unit", rx_unit);
+		JSON_ADD_ITEM_TO_OBJECT(iface, "rx", rx_json);
+
+		// Get IP address(es) of this interface
+		if(ifap)
+		{
+			// Walk through linked list of interface addresses
+			cJSON *ipv4 = JSON_NEW_ARRAY();
+			cJSON *ipv6 = JSON_NEW_ARRAY();
+			for(struct ifaddrs *ifa = ifap; ifa != NULL; ifa = ifa->ifa_next)
+			{
+				// Skip interfaces without an address and those
+				// not matching the current interface
+				if(ifa->ifa_addr == NULL || strcmp(ifa->ifa_name, iface_name) != 0)
+					continue;
+
+				// If we reach this point, we found the correct interface
+				const sa_family_t family = ifa->ifa_addr->sa_family;
+				char host[NI_MAXHOST] = { 0 };
+				if(family == AF_INET || family == AF_INET6)
+				{
+					// Get IP address
+					const int s = getnameinfo(ifa->ifa_addr,
+					                          (family == AF_INET) ?
+					                               sizeof(struct sockaddr_in) :
+					                               sizeof(struct sockaddr_in6),
+					                          host, NI_MAXHOST,
+					                          NULL, 0, NI_NUMERICHOST);
+					if (s != 0)
+					{
+						log_warn("API: getnameinfo() failed: %s\n", gai_strerror(s));
+						continue;
+					}
+
+					if(family == AF_INET)
+					{
+						JSON_COPY_STR_TO_ARRAY(ipv4, host);
+					}
+					else if(family == AF_INET6)
+					{
+						JSON_COPY_STR_TO_ARRAY(ipv6, host);
+					}
+				}
+			}
+			JSON_ADD_ITEM_TO_OBJECT(iface, "ipv4", ipv4);
+			JSON_ADD_ITEM_TO_OBJECT(iface, "ipv6", ipv6);
+		}
+
+		// Sum up transmitted and received bytes
+		if(tx_bytes > 0)
+			tx_sum += tx_bytes;
+		if(rx_bytes > 0)
+			rx_sum += rx_bytes;
+
+		// Add interface to array
+		JSON_ADD_ITEM_TO_ARRAY(interfaces, iface);
+	}
+
+	freeifaddrs(ifap);
+	closedir(dfd);
+
+	cJSON *sum = JSON_NEW_OBJECT();
+	JSON_COPY_STR_TO_OBJECT(sum, "name", "sum");
+	JSON_ADD_BOOL_TO_OBJECT(sum, "carrier", true);
+	JSON_ADD_NUMBER_TO_OBJECT(sum, "speed", 0);
+
+	// Format transmitted bytes
+	double tx = 0.0;
+	char tx_unit[3] = { 0 };
+	format_memory_size(tx_unit, tx_sum, &tx);
+	if(tx_unit[0] != '\0')
+		tx_unit[1] = 'B';
+
+	// Add transmitted bytes to interface record
+	cJSON *tx_json = JSON_NEW_OBJECT();
+	JSON_ADD_NUMBER_TO_OBJECT(tx_json, "num", tx);
+	JSON_COPY_STR_TO_OBJECT(tx_json, "unit", tx_unit);
+	JSON_ADD_ITEM_TO_OBJECT(sum, "tx", tx_json);
+
+	// Format received bytes
+	double rx = 0.0;
+	char rx_unit[3] = { 0 };
+	format_memory_size(rx_unit, rx_sum, &rx);
+	if(rx_unit[0] != '\0')
+		rx_unit[1] = 'B';
+
+	// Add received bytes to JSON object
+	cJSON *rx_json = JSON_NEW_OBJECT();
+	JSON_ADD_NUMBER_TO_OBJECT(rx_json, "num", rx);
+	JSON_COPY_STR_TO_OBJECT(rx_json, "unit", rx_unit);
+	JSON_ADD_ITEM_TO_OBJECT(sum, "rx", rx_json);
+
+	cJSON *ipv4 = JSON_NEW_ARRAY();
+	cJSON *ipv6 = JSON_NEW_ARRAY();
+	JSON_ADD_ITEM_TO_OBJECT(sum, "ipv4", ipv4);
+	JSON_ADD_ITEM_TO_OBJECT(sum, "ipv6", ipv6);
+
+	// Add interface to array
+	JSON_ADD_ITEM_TO_ARRAY(interfaces, sum);
+	JSON_ADD_ITEM_TO_OBJECT(json, "interfaces", interfaces);
+	JSON_SEND_OBJECT(json);
+}
+
+static int api_network_devices_GET(struct ftl_conn *api)
+{
+	// Does the user request a custom number of devices to be included?
+	unsigned int device_count = 10;
+	get_uint_var(api->request->query_string, "max_devices", &device_count);
+
+	// Does the user request a custom number of addresses per device to be included?
+	unsigned int address_count = 3;
+	get_uint_var(api->request->query_string, "max_addresses", &address_count);
+
+	// Open pihole-FTL.db database file
+	sqlite3_stmt *device_stmt = NULL, *ip_stmt = NULL;
+	sqlite3 *db = dbopen(true, false);
+	if(db == NULL)
+	{
+		log_warn("Failed to open database in networkTable_readDevices()");
+		return false;
+	}
+
+	const char *sql_msg = NULL;
+	if(!networkTable_readDevices(db, &device_stmt, &sql_msg))
+	{
+		// Add SQL message (may be NULL = not available)
+		return send_json_error(api, 500,
+		                       "database_error",
+		                       "Could not read network details from database table",
+		                       sql_msg);
+	}
+
+	// Read record for a single device
+	cJSON *devices = JSON_NEW_ARRAY();
+	network_record network;
+	unsigned int device_counter = 0;
+	while(networkTable_readDevicesGetRecord(device_stmt, &network, &sql_msg) &&
+	      device_counter++ < device_count)
+	{
+		cJSON *item = JSON_NEW_OBJECT();
+		JSON_ADD_NUMBER_TO_OBJECT(item, "id", network.id);
+		JSON_COPY_STR_TO_OBJECT(item, "hwaddr", network.hwaddr);
+		JSON_COPY_STR_TO_OBJECT(item, "interface", network.iface);
+		JSON_ADD_NUMBER_TO_OBJECT(item, "firstSeen", network.firstSeen);
+		JSON_ADD_NUMBER_TO_OBJECT(item, "lastQuery", network.lastQuery);
+		JSON_ADD_NUMBER_TO_OBJECT(item, "numQueries", network.numQueries);
+		JSON_COPY_STR_TO_OBJECT(item, "macVendor", network.macVendor);
+
+		// Build array of all IP addresses known associated to this client
+		cJSON *ips = JSON_NEW_ARRAY();
+		if(networkTable_readIPs(db, &ip_stmt, network.id, &sql_msg))
+		{
+			// Walk known IP addresses + names
+			network_addresses_record network_address;
+			unsigned int address_counter = 0;
+			while(networkTable_readIPsGetRecord(ip_stmt, &network_address, &sql_msg) &&
+			      address_counter++ < address_count)
+			{
+				cJSON *ip = JSON_NEW_OBJECT();
+				JSON_COPY_STR_TO_OBJECT(ip, "ip", network_address.ip);
+				JSON_COPY_STR_TO_OBJECT(ip, "name", network_address.name);
+				JSON_ADD_NUMBER_TO_OBJECT(ip, "lastSeen", network_address.lastSeen);
+				JSON_ADD_NUMBER_TO_OBJECT(ip, "nameUpdated", network_address.nameUpdated);
+				JSON_ADD_ITEM_TO_ARRAY(ips, ip);
+			}
+
+			// Possible error handling
+			if(sql_msg != NULL)
+			{
+				cJSON_Delete(ips);
+				cJSON_Delete(devices);
+				return send_json_error(api, 500,
+				                       "database_error",
+				                       "Could not read network details from database table (getting IP records)",
+				                       sql_msg);
+			}
+
+			// Finalize sub-query
+			networkTable_readIPsFinalize(ip_stmt);
+		}
+
+		// Add array of IP addresses to device
+		JSON_ADD_ITEM_TO_OBJECT(item, "ips", ips);
+
+		// Add device to array of all devices
+		JSON_ADD_ITEM_TO_ARRAY(devices, item);
+	}
+
+	if(sql_msg != NULL)
+	{
+		cJSON_Delete(devices);
+		return send_json_error(api, 500,
+		                       "database_error",
+		                       "Could not read network details from database table (step)",
+		                       sql_msg);
+	}
+
+	// Finalize query
+	networkTable_readDevicesFinalize(device_stmt);
+	dbclose(&db);
+
+	// Return data to user
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_ADD_ITEM_TO_OBJECT(json, "devices", devices);
+	JSON_SEND_OBJECT(json);
+}
+
+static int api_network_devices_DELETE(struct ftl_conn *api)
+{
+	// Get device ID
+	int device_id = 0;
+	if(sscanf(api->item, "%i", &device_id) != 1)
+	{
+		return send_json_error(api, 400,
+		                       "invalid_request",
+		                       "Missing or invalid {id} parameter",
+		                       NULL);
+	}
+
+	// Open pihole-FTL.db database file
+	sqlite3 *db = dbopen(false, false);
+	if(db == NULL)
+	{
+		log_warn("Failed to open database in networkTable_readDevices()");
+		return false;
+	}
+
+	// Delete row from network table by ID
+	const char *sql_msg = NULL;
+	int deleted = 0;
+	if(!networkTable_deleteDevice(db, device_id, &deleted, &sql_msg))
+	{
+		// Add SQL message (may be NULL = not available)
+		return send_json_error(api, 500,
+		                       "database_error",
+		                       "Could not delete network details from database table",
+		                       sql_msg);
+	}
+
+	// Close database
+	dbclose(&db);
+
+	// Send empty reply with codes:
+	// - 204 No Content (if any items were deleted)
+	// - 404 Not Found (if no items were deleted)
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_SEND_OBJECT_CODE(json, deleted > 0 ? 204 : 404);
+}
+
+int api_network_devices(struct ftl_conn *api)
+{
+	if(api->method == HTTP_GET)
+	{
+		return api_network_devices_GET(api);
+	}
+	else if(api->method == HTTP_DELETE)
+	{
+		return api_network_devices_DELETE(api);
+	}
+	else
+	{
+		return send_json_error(api, 405,
+		                       "method_not_allowed",
+		                       "Method not allowed",
+		                       NULL);
+	}
+}
+
+int api_client_suggestions(struct ftl_conn *api)
+{
+	// Get client suggestions
+	if(api->method != HTTP_GET)
+	{
+		// This results in error 404
+		return 0;
+	}
+
+	// Does the user request a custom number of addresses per device to be included?
+	unsigned int count = 50;
+	get_uint_var(api->request->query_string, "count", &count);
+
+	bool ipv4_only = true;
+	get_bool_var(api->request->query_string, "ipv4_only", &ipv4_only);
+
+	// Open pihole-FTL.db database file connection
+	sqlite3 *db = dbopen(true, false);
+
+	// Attach gravity database
+	const char *message = "";
+	if(!attach_database(db, &message, config.files.gravity.v.s, "g"))
+	{
+		log_err("Failed to attach gravity database: %s", message);
+		dbclose(&db);
+		return send_json_error(api, 500,
+		                       "database_error",
+		                       "Could not attach gravity database",
+		                       message);
+	}
+
+	// Prepare SQL statement
+	sqlite3_stmt *stmt = NULL;
+	const char *sql = "SELECT n.hwaddr,n.macVendor,n.lastQuery,"
+	                  "(SELECT GROUP_CONCAT(DISTINCT na.ip) "
+	                    "FROM network_addresses na "
+	                      "WHERE na.network_id = n.id),"
+	                  "(SELECT GROUP_CONCAT(DISTINCT na.name) "
+	                    "FROM network_addresses na "
+	                      "WHERE na.network_id = n.id) "
+	                  "FROM network n "
+	                  "ORDER BY lastQuery DESC LIMIT ?";
+
+	if(sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK)
+	{
+		log_err("Failed to prepare SQL statement: %s", sqlite3_errmsg(db));
+		dbclose(&db);
+		return send_json_error(api, 500,
+		                       "database_error",
+		                       "Could not prepare SQL statement",
+		                       sqlite3_errmsg(db));
+	}
+
+	// Bind parameters
+	if(sqlite3_bind_int(stmt, 1, count) != SQLITE_OK)
+	{
+		log_err("Failed to bind parameter: %s", sqlite3_errmsg(db));
+		sqlite3_finalize(stmt);
+		dbclose(&db);
+		return send_json_error(api, 500,
+		                       "database_error",
+		                       "Could not bind parameter",
+		                       sqlite3_errmsg(db));
+	}
+
+	// Execute SQL statement
+	cJSON *clients = JSON_NEW_ARRAY();
+	while(sqlite3_step(stmt) == SQLITE_ROW)
+	{
+		cJSON *client = JSON_NEW_OBJECT();
+		JSON_COPY_STR_TO_OBJECT(client, "hwaddr", sqlite3_column_text(stmt, 0));
+		JSON_COPY_STR_TO_OBJECT(client, "macVendor", sqlite3_column_text(stmt, 1));
+		JSON_ADD_NUMBER_TO_OBJECT(client, "lastQuery", sqlite3_column_int(stmt, 2));
+		JSON_COPY_STR_TO_OBJECT(client, "addresses", sqlite3_column_text(stmt, 3));
+		JSON_COPY_STR_TO_OBJECT(client, "names", sqlite3_column_text(stmt, 4));
+		JSON_ADD_ITEM_TO_ARRAY(clients, client);
+	}
+
+	// Finalize query
+	sqlite3_finalize(stmt);
+
+	// Detach gravity database
+	if(!detach_database(db, &message, "g"))
+	{
+		log_err("Failed to detach gravity database: %s", message);
+		dbclose(&db);
+		return send_json_error(api, 500,
+		                       "database_error",
+		                       "Could not detach gravity database",
+		                       message);
+	}
+
+	// Close database connection
+	dbclose(&db);
+
+	// Return data to user
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_ADD_ITEM_TO_OBJECT(json, "clients", clients);
+	JSON_SEND_OBJECT(json);
+}
+

src/api/request.c

@@ -1,205 +0,0 @@
-/* Pi-hole: A black hole for Internet advertisements
-*  (c) 2017 Pi-hole, LLC (https://pi-hole.net)
-*  Network-wide ad blocking via your own hardware.
-*
-*  FTL Engine
-*  Socket request handling routines
-*
-*  This file is copyright under the latest version of the EUPL.
-*  Please see LICENSE file for your rights under this license. */
-
-#include "../FTL.h"
-#include "api.h"
-#include "../shmem.h"
-#include "../timers.h"
-#include "request.h"
-#include "socket.h"
-#include "../resolve.h"
-#include "../regex_r.h"
-#include "../database/network-table.h"
-#include "../log.h"
-// Eventqueue routines
-#include "../events.h"
-#include "../config.h"
-
-bool __attribute__((pure)) command(const char *client_message, const char* cmd) {
-	return strstr(client_message, cmd) != NULL;
-}
-
-bool process_request(const char *client_message, const int sock, const bool istelnet)
-{
-	char EOT[2];
-	EOT[0] = 0x04;
-	EOT[1] = 0x00;
-	bool processed = false;
-
-	if(command(client_message, ">stats"))
-	{
-		processed = true;
-		lock_shm();
-		getStats(sock, istelnet);
-		unlock_shm();
-	}
-	else if(command(client_message, ">overTime"))
-	{
-		processed = true;
-		lock_shm();
-		getOverTime(sock, istelnet);
-		unlock_shm();
-	}
-	else if(command(client_message, ">top-domains") || command(client_message, ">top-ads"))
-	{
-		processed = true;
-		lock_shm();
-		getTopDomains(client_message, sock, istelnet);
-		unlock_shm();
-	}
-	else if(command(client_message, ">top-clients"))
-	{
-		processed = true;
-		lock_shm();
-		getTopClients(client_message, sock, istelnet);
-		unlock_shm();
-	}
-	else if(command(client_message, ">forward-dest"))
-	{
-		processed = true;
-		lock_shm();
-		getUpstreamDestinations(client_message, sock, istelnet);
-		unlock_shm();
-	}
-	else if(command(client_message, ">forward-names"))
-	{
-		processed = true;
-		lock_shm();
-		getUpstreamDestinations(">forward-dest unsorted", sock, istelnet);
-		unlock_shm();
-	}
-	else if(command(client_message, ">querytypes"))
-	{
-		processed = true;
-		lock_shm();
-		getQueryTypes(sock, istelnet);
-		unlock_shm();
-	}
-	else if(command(client_message, ">getallqueries"))
-	{
-		processed = true;
-		lock_shm();
-		getAllQueries(client_message, sock, istelnet);
-		unlock_shm();
-	}
-	else if(command(client_message, ">recentBlocked"))
-	{
-		processed = true;
-		lock_shm();
-		getRecentBlocked(client_message, sock, istelnet);
-		unlock_shm();
-	}
-	else if(command(client_message, ">clientID"))
-	{
-		processed = true;
-		lock_shm();
-		getClientID(sock, istelnet);
-		unlock_shm();
-	}
-	else if(command(client_message, ">version"))
-	{
-		processed = true;
-		// No lock required
-		getVersion(sock, istelnet);
-	}
-	else if(command(client_message, ">dbstats"))
-	{
-		processed = true;
-		// No lock required. Access to the database
-		// is guaranteed to be atomic
-		getDBstats(sock, istelnet);
-	}
-	else if(command(client_message, ">ClientsoverTime"))
-	{
-		processed = true;
-		lock_shm();
-		getClientsOverTime(sock, istelnet);
-		unlock_shm();
-	}
-	else if(command(client_message, ">client-names"))
-	{
-		processed = true;
-		lock_shm();
-		getClientNames(sock, istelnet);
-		unlock_shm();
-	}
-	else if(command(client_message, ">unknown"))
-	{
-		processed = true;
-		lock_shm();
-		getUnknownQueries(sock, istelnet);
-		unlock_shm();
-	}
-	else if(command(client_message, ">cacheinfo"))
-	{
-		processed = true;
-		lock_shm();
-		getCacheInformation(sock);
-		unlock_shm();
-	}
-	else if(command(client_message, ">reresolve"))
-	{
-		processed = true;
-		logg("Received API request to re-resolve host names");
-		set_event(RELOAD_PRIVACY_LEVEL);
-	}
-	else if(command(client_message, ">recompile-regex"))
-	{
-		processed = true;
-		logg("Received API request to recompile regex");
-		lock_shm();
-		// Reread regex.list
-		// Read and compile possible regex filters
-		read_regex_from_database();
-		unlock_shm();
-	}
-	else if(command(client_message, ">delete-lease"))
-	{
-		processed = true;
-		delete_lease(client_message, sock);
-	}
-	else if(command(client_message, ">dns-port"))
-	{
-		processed = true;
-		getDNSport(sock);
-	}
-	else if(command(client_message, ">maxlogage"))
-	{
-		processed = true;
-		getMAXLOGAGE(sock);
-	}
-	else if(command(client_message, ">gateway"))
-	{
-		processed = true;
-		getGateway(sock);
-	}
-	else if(command(client_message, ">interfaces"))
-	{
-		processed = true;
-		getInterfaces(sock);
-	}
-
-	// Test only at the end if we want to quit or kill
-	// so things can be processed before
-	if(command(client_message, ">quit") || command(client_message, EOT))
-	{
-		if(config.debug & DEBUG_API)
-			logg("Received >quit or EOT on socket %d", sock);
-		return true;
-	}
-
-	if(!processed)
-		ssend(sock, "unknown command: %s\n", client_message);
-
-	// End of queryable commands: Send EOM
-	seom(sock, istelnet);
-
-	return false;
-}

src/api/search.c

@@ -0,0 +1,324 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2023 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  API Implementation /api/search
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#include "FTL.h"
+#include "webserver/http-common.h"
+#include "webserver/json_macros.h"
+#include "api/api.h"
+#include "database/gravity-db.h"
+// match_regex()
+#include "regex_r.h"
+// parse_groupIDs()
+#include "webserver/http-common.h"
+#include <idn2.h>
+
+#define MAX_SEARCH_RESULTS 10000u
+
+static int search_table(struct ftl_conn *api, const char *item,
+                        const enum gravity_list_type listtype,
+                        char *ids, const unsigned int limit,
+                        unsigned int *N, const bool partial, cJSON* json)
+{
+	if(ids != NULL)
+	{
+		// Set item to NULL to indicate that we are searching for IDs
+		item = NULL;
+		// Strip "[" and "]" from ids
+		ids[strlen(ids)-1] = '\0';
+		ids++;
+	}
+
+	// Check domain against lists table
+	const char *sql_msg = NULL;
+	if(!gravityDB_readTable(listtype, item, &sql_msg, !partial, ids))
+	{
+		return send_json_error(api, 400, // 400 Bad Request
+		                       "database_error",
+		                       "Could not read domains from database table",
+		                       sql_msg);
+	}
+
+	tablerow table;
+	while(gravityDB_readTableGetRow(listtype, &table, &sql_msg))
+	{
+		if(++(*N) > limit)
+			continue;
+
+		cJSON *row = JSON_NEW_OBJECT();
+		JSON_COPY_STR_TO_OBJECT(row, "domain", table.domain);
+		if(table.type != NULL)
+			JSON_REF_STR_IN_OBJECT(row, "type", table.type);
+		if(table.kind != NULL)
+			JSON_REF_STR_IN_OBJECT(row, "kind", table.kind);
+		if(table.address != NULL)
+			JSON_COPY_STR_TO_OBJECT(row, "address", table.address);
+		JSON_COPY_STR_TO_OBJECT(row, "comment", table.comment);
+		JSON_ADD_BOOL_TO_OBJECT(row, "enabled", table.enabled);
+		// Add read-only database parameters
+		JSON_ADD_NUMBER_TO_OBJECT(row, "id", table.id);
+		JSON_ADD_NUMBER_TO_OBJECT(row, "date_added", table.date_added);
+		JSON_ADD_NUMBER_TO_OBJECT(row, "date_modified", table.date_modified);
+
+		if(listtype == GRAVITY_GRAVITY || listtype == GRAVITY_ANTIGRAVITY)
+		{
+			// Add gravity specific parameters
+			JSON_REF_STR_IN_OBJECT(row, "type", table.type);
+			JSON_ADD_NUMBER_TO_OBJECT(row, "date_updated", table.date_updated);
+			JSON_ADD_NUMBER_TO_OBJECT(row, "number", table.number);
+			JSON_ADD_NUMBER_TO_OBJECT(row, "invalid_domains", table.invalid_domains);
+			JSON_ADD_NUMBER_TO_OBJECT(row, "abp_entries", table.abp_entries);
+			JSON_ADD_NUMBER_TO_OBJECT(row, "status", table.status);
+		}
+
+		if(table.group_ids != NULL)
+		{
+			const int ret = parse_groupIDs(api, &table, row);
+			if(ret != 0)
+				return ret;
+		}
+		else
+		{
+			// Empty group set
+			cJSON *group_ids = JSON_NEW_ARRAY();
+			JSON_ADD_ITEM_TO_OBJECT(row, "groups", group_ids);
+		}
+		JSON_ADD_ITEM_TO_ARRAY(json, row);
+	}
+	gravityDB_readTableFinalize();
+
+	return 200;
+}
+
+static int search_gravity(struct ftl_conn *api, const char *punycode, cJSON *array,
+                          cJSON **abp_patterns, const unsigned int limit, unsigned int *N,
+                          const bool partial, const bool antigravity)
+{
+	enum gravity_list_type table = antigravity ? GRAVITY_ANTIGRAVITY : GRAVITY_GRAVITY;
+	if(partial)
+	{
+		// Search for partial matches in (anti/)gravity
+		const int ret = search_table(api, punycode, table, NULL, limit, N, partial, array);
+		if(ret != 200)
+			return ret;
+	}
+	else
+	{
+		// Search for exact matches in (anti/)gravity
+		int ret = search_table(api, punycode, table, NULL, limit, N, false, array);
+		if(ret != 200)
+			return ret;
+
+		// Search for ABP matches in (anti/)gravity
+		*abp_patterns = gen_abp_patterns(punycode, antigravity);
+		cJSON *abp_pattern = NULL;
+		cJSON_ArrayForEach(abp_pattern, *abp_patterns)
+		{
+			const char *pattern = cJSON_GetStringValue(abp_pattern);
+			if(pattern == NULL)
+				continue;
+			ret = search_table(api, pattern, table, NULL, limit, N, partial, array);
+			if(ret != 200)
+				return ret;
+		}
+	}
+
+	return 200;
+}
+
+int api_search(struct ftl_conn *api)
+{
+	int ret = 0;
+	const char *domain = api->item;
+	if(domain == NULL || strlen(domain) == 0)
+	{
+		// No search term provided
+		return send_json_error(api, 400,
+		                       "bad_request",
+		                       "Invalid request: No search term provided",
+		                       api->request->local_uri_raw);
+	}
+
+	// Parse query string parameters
+	bool partial = false, debug = false;
+	unsigned int limit = 20u;
+	if(api->request->query_string != NULL)
+	{
+		// Check if we should perform a partial search
+		get_bool_var(api->request->query_string, "partial", &partial);
+		get_bool_var(api->request->query_string, "debug", &debug);
+		get_uint_var(api->request->query_string, "N", &limit);
+
+		// Check validity of limit
+		if(limit > MAX_SEARCH_RESULTS)
+		{
+			// Too many results requested
+			char hint[100];
+			sprintf(hint, "Requested %u number of results but hard upper limit is %u", limit, MAX_SEARCH_RESULTS);
+			return send_json_error(api, 400,
+			                       "bad_request",
+			                       "Invalid request: Requested too many results",
+			                       hint);
+		}
+	}
+
+	// Convert domain to punycode
+	// The IDNA document defines internationalized domain names (IDNs) and a
+	// mechanism called IDNA for handling them in a standard fashion. IDNs
+	// use characters drawn from a large repertoire (Unicode), but IDNA
+	// allows the non-ASCII characters to be represented using only the
+	// ASCII characters already allowed in so-called host names today.
+	// idn2_to_ascii_lz() convert domain name in the locale’s encoding to an
+	// ASCII string. The domain name may contain several labels, separated
+	// by dots. The output buffer must be deallocated by the caller.
+	// Used flags:
+	// - IDN2_NFC_INPUT: Input is in Unicode Normalization Form C (NFC)
+	// - IDN2_NONTRANSITIONAL: Use Unicode TR46 non-transitional processing
+	char *punycode = NULL;
+	const int rc = idn2_to_ascii_lz(domain, &punycode, IDN2_NFC_INPUT | IDN2_NONTRANSITIONAL);
+	if (rc != IDN2_OK)
+	{
+		// Invalid domain name
+		return send_json_error(api, 400,
+		                       "bad_request",
+		                       "Invalid request: Invalid domain name",
+		                       idn2_strerror(rc));
+	}
+
+	// Convert punycode domain to lowercase
+	for(unsigned int i = 0u; i < strlen(punycode); i++)
+		punycode[i] = tolower(punycode[i]);
+
+	// Search through all exact domains
+	cJSON *domains = JSON_NEW_ARRAY();
+	unsigned int Nexact = 0u;
+	ret = search_table(api, punycode, GRAVITY_DOMAINLIST_ALL_EXACT, NULL, limit, &Nexact, partial, domains);
+	if(ret != 200)
+	{
+		free(punycode);
+		return ret;
+	}
+
+	// Search through gravity
+	cJSON *gravity = JSON_NEW_ARRAY();
+	cJSON *gravity_patterns = NULL;
+	unsigned int Ngravity = 0u;
+	ret = search_gravity(api, punycode, gravity, &gravity_patterns, limit, &Ngravity, partial, false);
+	if(ret != 200)
+	{
+		free(punycode);
+		return ret;
+	}
+
+	// Search through antigravity
+	cJSON *antigravity_patterns = NULL;
+	unsigned int Nantigravity = 0u;
+	ret = search_gravity(api, punycode, gravity, &antigravity_patterns, limit, &Nantigravity, partial, true);
+	if(ret != 200)
+	{
+		free(punycode);
+		return ret;
+	}
+
+	// Search through all regex filters
+	cJSON *regex_ids = JSON_NEW_OBJECT();
+	check_all_regex(punycode, regex_ids);
+	cJSON *deny_ids = cJSON_GetObjectItem(regex_ids, "deny");
+	cJSON *allow_ids = cJSON_GetObjectItem(regex_ids, "allow");
+
+	// Get allow regex filters
+	unsigned int Nregex = 0u;
+	if(cJSON_GetArraySize(allow_ids) > 0)
+	{
+		char *allow_list = cJSON_PrintUnformatted(allow_ids);
+		ret = search_table(api,punycode, GRAVITY_DOMAINLIST_ALLOW_REGEX, allow_list, limit, &Nregex, false, domains);
+		free(allow_list);
+		if(ret != 200)
+		{
+			free(punycode);
+			return ret;
+		}
+	}
+
+	if(cJSON_GetArraySize(deny_ids) > 0)
+	{
+		char *deny_list = cJSON_PrintUnformatted(deny_ids);
+		ret = search_table(api, punycode, GRAVITY_DOMAINLIST_DENY_REGEX, deny_list, limit, &Nregex, false, domains);
+		free(deny_list);
+		if(ret != 200)
+		{
+			free(punycode);
+			return ret;
+		}
+	}
+
+	cJSON *search = JSON_NEW_OBJECT();
+	// .domains.{}
+	JSON_ADD_ITEM_TO_OBJECT(search, "domains", domains);
+	// .gravity.{}
+	JSON_ADD_ITEM_TO_OBJECT(search, "gravity", gravity);
+
+	// .results.{}
+	cJSON *results = JSON_NEW_OBJECT();
+
+	// .results.domains.{}
+	cJSON *jdomains = JSON_NEW_OBJECT();
+	JSON_ADD_NUMBER_TO_OBJECT(jdomains, "exact", Nexact);
+	JSON_ADD_NUMBER_TO_OBJECT(jdomains, "regex", Nregex);
+	JSON_ADD_ITEM_TO_OBJECT(results, "domains", jdomains);
+
+	// .results.gravity.{}
+	cJSON *jgravity = JSON_NEW_OBJECT();
+	JSON_ADD_NUMBER_TO_OBJECT(jgravity, "allow", Nantigravity);
+	JSON_ADD_NUMBER_TO_OBJECT(jgravity, "block", Ngravity);
+	JSON_ADD_ITEM_TO_OBJECT(results, "gravity", jgravity);
+
+	// .results.total
+	JSON_ADD_NUMBER_TO_OBJECT(results, "total", Nexact+Nregex+Ngravity+Nantigravity);
+	JSON_ADD_ITEM_TO_OBJECT(search, "results", results);
+
+	// .parameters.{}
+	cJSON *parameters = JSON_NEW_OBJECT();
+	JSON_ADD_NUMBER_TO_OBJECT(parameters, "N", limit);
+	JSON_ADD_BOOL_TO_OBJECT(parameters, "partial", partial);
+	JSON_REF_STR_IN_OBJECT(parameters, "domain", api->item);
+	JSON_ADD_BOOL_TO_OBJECT(parameters, "debug", debug);
+	JSON_ADD_ITEM_TO_OBJECT(search, "parameters", parameters);
+
+	// .debug.{}
+	if(debug)
+	{
+		// Add debug information
+		cJSON *abp_pattern = JSON_NEW_OBJECT();
+		JSON_ADD_ITEM_TO_OBJECT(abp_pattern, "gravity", gravity_patterns);
+		JSON_ADD_ITEM_TO_OBJECT(abp_pattern, "antigravity", antigravity_patterns);
+		cJSON *jdebug = JSON_NEW_OBJECT();
+		JSON_COPY_STR_TO_OBJECT(jdebug, "domain", domain);
+		JSON_COPY_STR_TO_OBJECT(jdebug, "punycode", punycode);
+		JSON_ADD_ITEM_TO_OBJECT(jdebug, "abp_pattern", abp_pattern);
+		JSON_ADD_ITEM_TO_OBJECT(jdebug, "regex_ids", regex_ids);
+		JSON_ADD_ITEM_TO_OBJECT(search, "debug", jdebug);
+	}
+	else
+	{
+		// Free intermediate JSON objects containing ABP patterns
+		cJSON_Delete(gravity_patterns);
+		cJSON_Delete(antigravity_patterns);
+
+		// Free intermediate JSON objects containing list of regex IDs
+		cJSON_Delete(regex_ids);
+	}
+
+	// Free punycode
+	free(punycode);
+
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_ADD_ITEM_TO_OBJECT(json, "search", search);
+	JSON_SEND_OBJECT(json);
+}

src/api/socket.c

@@ -1,271 +0,0 @@
-/* Pi-hole: A black hole for Internet advertisements
-*  (c) 2017 Pi-hole, LLC (https://pi-hole.net)
-*  Network-wide ad blocking via your own hardware.
-*
-*  FTL Engine
-*  Socket connection routines
-*
-*  This file is copyright under the latest version of the EUPL.
-*  Please see LICENSE file for your rights under this license. */
-
-#include "FTL.h"
-#include "api.h"
-#include "../log.h"
-#include "socket.h"
-#include "request.h"
-#include "../config.h"
-// sleepms()
-#include "../timers.h"
-// global variable killed
-#include "../signals.h"
-// API thread storage
-#include "../daemon.h"
-#include "../shmem.h"
-
-// The backlog argument defines the maximum length
-// to which the queue of pending connections for
-// telnetfd may grow. If a connection request arrives
-// when the queue is full, the client may receive an
-// error with an indication of ECONNREFUSED or, if
-// the underlying protocol supports retransmission,
-// the request may be ignored so that a later
-// reattempt at connection succeeds.
-#define BACKLOG 5
-
-static int bind_to_telnet_socket(const enum telnet_type type, const char *stype)
-{
-	const int socketdescriptor = socket(type == TELNET_SOCK ? AF_LOCAL : (type == TELNETv4 ? AF_INET : AF_INET6), SOCK_STREAM, 0);
-	if(socketdescriptor < 0)
-	{
-		logg("Error opening %s telnet socket: %s (%i)", stype, strerror(errno), errno);
-		return -1;
-	}
-
-	const size_t addrlen = MAX(sizeof(struct sockaddr_un), MAX(sizeof(struct sockaddr_in), sizeof(struct sockaddr_in6)));
-	void *address = calloc(1, addrlen);
-
-	if(type == TELNETv4 || type == TELNETv6)
-	{
-		// Set SO_REUSEADDR to allow re-binding to the port that has been used
-		// previously by FTL. A common pattern is that you change FTL's
-		// configuration file and need to restart that server to make it reload
-		// its configuration. Without SO_REUSEADDR, the bind() call in the restarted
-		// new instance will fail if there were connections open to the previous
-		// instance when you killed it. Those connections will hold the TCP port in
-		// the TIME_WAIT state for 30-120 seconds, so you fall into case 1 above.
-		if(setsockopt(socketdescriptor, SOL_SOCKET, SO_REUSEADDR, &(int){ 1 }, sizeof(int)) != 0)
-			logg("WARN: allowing re-binding (%s) failed: %s", stype, strerror(errno));
-
-		if(type == TELNETv6)
-		{
-			// If this flag is set to true (nonzero), then the  socket  is  re‐
-			// stricted  to  sending  and receiving IPv6 packets only.  In this
-			// case, an IPv4 and an IPv6 application can bind to a single  port
-			// at the same time.
-			if(setsockopt(socketdescriptor, IPPROTO_IPV6, IPV6_V6ONLY, &(int){ 1 }, sizeof(int)) != 0)
-				logg("WARN: setting socket to IPv6-only failed: %s", strerror(errno));
-
-			if(config.socket_listenlocal)
-				((struct sockaddr_in6*) address)->sin6_addr = in6addr_loopback;
-			else
-				((struct sockaddr_in6*) address)->sin6_addr = in6addr_any;
-
-			// The bind() system call binds a socket to an address,
-			// in this case the address of the current host and
-			// port number on which the server will run.
-			// convert this to network byte order using the function htons()
-			// which converts a port number in host byte order to a port number
-			// in network byte order
-
-			// Bind to IPv6 socket
-			((struct sockaddr_in6*) address)->sin6_family = AF_INET6;
-			((struct sockaddr_in6*) address)->sin6_port = htons(config.port);
-		}
-		else // IPv4
-		{
-
-			if(config.socket_listenlocal)
-				((struct sockaddr_in*) address)->sin_addr.s_addr = htonl(INADDR_LOOPBACK);
-			else
-				((struct sockaddr_in*) address)->sin_addr.s_addr = INADDR_ANY;
-
-			// Bind to IPv4 port
-			((struct sockaddr_in*) address)->sin_family = AF_INET;
-			((struct sockaddr_in*) address)->sin_port = htons(config.port);
-		}
-	}
-	else // socket
-	{
-		// Make sure unix socket file handle does not exist, if it exists, remove it
-		unlink(FTLfiles.socketfile);
-
-		((struct sockaddr_un*) address)->sun_family = AF_LOCAL;
-		// The sockaddr_un.sum_path may be shorter than the size of the FTLfiles.socketfile
-		// buffer. Ensure that the string is null-terminated even when the string is too large.
-		// In case strlen(FTLfiles.socketfile) < sizeof(address.sun_path) [this will virtually
-		// always be the case], the explicit setting of the last byte to zero is a no-op as
-		// strncpy() writes additional null bytes to ensure that a total of n bytes are written.
-		strncpy(((struct sockaddr_un*) address)->sun_path, FTLfiles.socketfile, sizeof(((struct sockaddr_un*) address)->sun_path));
-		((struct sockaddr_un*) address)->sun_path[sizeof(((struct sockaddr_un*) address)->sun_path)-1] = '\0';
-
-	}
-
-	// Bind to socket
-	if(bind(socketdescriptor, (struct sockaddr *) address, addrlen) < 0)
-	{
-		logg("Error binding to %s telnet socket: %s (%i)", stype, strerror(errno), errno);
-		return -1;
-	}
-
-	// The listen system call allows the process to listen on the socket for connections
-	if(listen(socketdescriptor, BACKLOG) == -1)
-	{
-		logg("Error listening on %s telnet socket: %s (%i)", stype, strerror(errno), errno);
-		return -1;
-	}
-
-	logg("Listening on port %i for incoming %s telnet connections", config.port, stype);
-	return socketdescriptor;
-}
-
-static void *telnet_connection_handler_thread(void *args)
-{
-	struct thread_info *tinfo = args;
-	// Set thread name
-	char threadname[16] = { 0 };
-	snprintf(threadname, sizeof(threadname), "telnet-%s-%i", tinfo->stype, tinfo->tid);
-	prctl(PR_SET_NAME, threadname, 0, 0, 0);
-
-	// Ensure this thread can be canceled at any time (not only at
-	// cancellation points)
-	pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
-
-	if(config.debug & DEBUG_API)
-		logg("Started telnet thread %s", threadname);
-
-	// Listen as long as this thread is not canceled
-	int errors = 0;
-	while(!killed)
-	{
-		// Look for new clients that want to connect
-		const int csck = accept(tinfo->fd, NULL, NULL);
-		if(csck == -1)
-		{
-			logg("Telnet error in %s: %s (%i, fd: %d)", threadname, strerror(errno), errno, tinfo->fd);
-			if(errors++ > 20)
-				break;
-			sleepms(100);
-			continue;
-		}
-
-		// Define buffer for client's message
-		char client_message[SOCKETBUFFERLEN] ={ 0 };
-
-		// Receive from client
-		ssize_t n;
-		while((n = recv(csck, client_message, SOCKETBUFFERLEN-1, 0)))
-		{
-			if (n > 0 && n < SOCKETBUFFERLEN)
-			{
-				// Null-terminate client string
-				client_message[n] = '\0';
-				char *message = strdup(client_message);
-				if(message == NULL)
-				{
-					if(config.debug & DEBUG_API)
-						logg("Break in telnet thread for socket %d/%d: Memory error", tinfo->fd, csck);
-					break;
-				}
-
-				// Clear client message receive buffer
-				memset(client_message, 0, sizeof client_message);
-
-				// Process received message
-				const bool eom = process_request(message, csck, tinfo->istelnet);
-				free(message);
-				if(eom) break;
-			}
-			else if(n == -1)
-			{
-				if(config.debug & DEBUG_API)
-					logg("Break in telnet thread for socket %d/%d: No data received", tinfo->fd, csck);
-				break;
-			}
-		}
-
-		// Close client socket
-		close(csck);
-	}
-
-	if(config.debug & DEBUG_API)
-		logg("Terminating telnet thread %s (%d errors)", threadname, errors);
-
-	// Free thread-private memory
-	free(tinfo);
-	return NULL;
-}
-
-void listen_telnet(const enum telnet_type type)
-{
-	// We will use the attributes object later to start all threads in detached mode
-	pthread_attr_t attr;
-	// Initialize thread attributes object with default attribute values
-	pthread_attr_init(&attr);
-	// When a detached thread terminates, its resources are automatically released back to
-	// the system without the need for another thread to join with the terminated thread
-	pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
-
-	// Initialize telnet socket
-	const char *stype = type == TELNET_SOCK ? "socket" : (type == TELNETv4 ? "IPv4" : "IPv6");
-	const int fd = bind_to_telnet_socket(type, stype);
-	if(fd < 0)
-	{
-		logg("WARN: Cannot bind to %s telnet socket", stype);
-		return;
-	}
-
-	if(config.debug & DEBUG_API)
-		logg("Telnet-%s listener accepting on fd %d", stype, fd);
-
-	for(unsigned int i = 0; i < MAX_API_THREADS; i++)
-	{
-		// Spawn telnet thread
-		// Create a private copy of the socket fd for the child thread
-		struct thread_info *tinfo = calloc(1, sizeof(struct thread_info));
-		if(!tinfo)
-			continue;
-
-		tinfo->fd = fd;
-		tinfo->tid = i;
-		tinfo->istelnet = (type == TELNETv4 || type == TELNETv6);
-		tinfo->stype = stype;
-		if(pthread_create(&api_threads[i], &attr, telnet_connection_handler_thread, (void*) tinfo) != 0)
-		{
-			// Log the error code description
-			logg("WARNING: Unable to open telnet processing thread: %s", strerror(errno));
-		}
-	}
-}
-
-void seom(const int sock, const bool istelnet)
-{
-	if(istelnet)
-		ssend(sock, "---EOM---\n\n");
-	else
-		pack_eom(sock);
-}
-
-bool __attribute__ ((format (gnu_printf, 5, 6))) _ssend(const int sock, const char *file, const char *func, const int line, const char *format, ...)
-{
-	char *buffer;
-	va_list args;
-	va_start(args, format);
-	int bytes = vasprintf(&buffer, format, args);
-	va_end(args);
-	if(bytes > 0 && buffer != NULL)
-	{
-		FTLwrite(sock, buffer, bytes, short_path(file), func, line);
-		free(buffer);
-	}
-	return errno == 0;
-}

src/api/socket.h

@@ -1,29 +0,0 @@
-/* Pi-hole: A black hole for Internet advertisements
-*  (c) 2019 Pi-hole, LLC (https://pi-hole.net)
-*  Network-wide ad blocking via your own hardware.
-*
-*  FTL Engine
-*  Socket prototypes
-*
-*  This file is copyright under the latest version of the EUPL.
-*  Please see LICENSE file for your rights under this license. */
-#ifndef SOCKET_H
-#define SOCKET_H
-
-// enum telnet_type
-#include "../enums.h"
-
-struct thread_info {
-	int fd;
-	int tid;
-	bool istelnet;
-	const char *stype;
-};
-
-void close_unix_socket(bool unlink_file);
-void seom(const int sock, const bool istelnet);
-#define ssend(sock, format, ...) _ssend(sock, __FILE__, __FUNCTION__,  __LINE__, format, ##__VA_ARGS__)
-bool _ssend(const int sock, const char *file, const char *func, const int line, const char *format, ...) __attribute__ ((format (gnu_printf, 5, 6)));
-void listen_telnet(const enum telnet_type type);
-
-#endif //SOCKET_H

src/api/stats.c

@@ -0,0 +1,649 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2019 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  API Implementation
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#include "FTL.h"
+#include "webserver/http-common.h"
+#include "webserver/json_macros.h"
+#include "api/api.h"
+#include "shmem.h"
+#include "datastructure.h"
+// read_setupVarsconf()
+#include "config/setupVars.h"
+// logging routines
+#include "log.h"
+// config struct
+#include "config/config.h"
+// overTime data
+#include "overTime.h"
+// enum REGEX
+#include "regex_r.h"
+// sqrt()
+#include <math.h>
+
+/* qsort comparison function (count field), sort ASC
+static int __attribute__((pure)) cmpasc(const void *a, const void *b)
+{
+	const int *elem1 = (int*)a;
+	const int *elem2 = (int*)b;
+
+	if (elem1[1] < elem2[1])
+		return -1;
+	else if (elem1[1] > elem2[1])
+		return 1;
+	else
+		return 0;
+} */
+
+// qsort subroutine, sort DESC
+int __attribute__((pure)) cmpdesc(const void *a, const void *b)
+{
+	const int *elem1 = (int*)a;
+	const int *elem2 = (int*)b;
+
+	if (elem1[1] > elem2[1])
+		return -1;
+	else if (elem1[1] < elem2[1])
+		return 1;
+	else
+		return 0;
+}
+
+static int get_query_types_obj(struct ftl_conn *api, cJSON *types)
+{
+	for(unsigned int i = TYPE_A; i < TYPE_MAX; i++)
+	{
+		// We add the collective OTHER type at the end
+		if(i == TYPE_OTHER)
+			continue;
+		JSON_ADD_NUMBER_TO_OBJECT(types, get_query_type_str(i, NULL, NULL), counters->querytype[i]);
+	}
+	JSON_ADD_NUMBER_TO_OBJECT(types, "OTHER", counters->querytype[TYPE_OTHER]);
+
+	return 0;
+}
+
+int api_stats_summary(struct ftl_conn *api)
+{
+	const int blocked =  get_blocked_count();
+	const int forwarded =  get_forwarded_count();
+	const int cached =  get_cached_count();
+	const int total = counters->queries;
+	float percent_blocked = 0.0f;
+
+	// Avoid 1/0 condition
+	if(total > 0)
+		percent_blocked = 1e2f*blocked/total;
+
+	// Lock shared memory
+	lock_shm();
+
+	cJSON *queries = JSON_NEW_OBJECT();
+	JSON_ADD_NUMBER_TO_OBJECT(queries, "total", total);
+	JSON_ADD_NUMBER_TO_OBJECT(queries, "blocked", blocked);
+	JSON_ADD_NUMBER_TO_OBJECT(queries, "percent_blocked", percent_blocked);
+	JSON_ADD_NUMBER_TO_OBJECT(queries, "unique_domains", counters->domains);
+	JSON_ADD_NUMBER_TO_OBJECT(queries, "forwarded", forwarded);
+	JSON_ADD_NUMBER_TO_OBJECT(queries, "cached", cached);
+
+	cJSON *types = JSON_NEW_OBJECT();
+	int ret = get_query_types_obj(api, types);
+	if(ret != 0)
+		return ret;
+	JSON_ADD_ITEM_TO_OBJECT(queries, "types", types);
+
+	cJSON *statuses = JSON_NEW_OBJECT();
+	for(enum query_status status = 0; status < QUERY_STATUS_MAX; status++)
+		JSON_ADD_NUMBER_TO_OBJECT(statuses, get_query_status_str(status), counters->status[status]);
+	JSON_ADD_ITEM_TO_OBJECT(queries, "status", statuses);
+
+	cJSON *replies = JSON_NEW_OBJECT();
+	for(enum reply_type reply = 0; reply <QUERY_REPLY_MAX; reply++)
+		JSON_ADD_NUMBER_TO_OBJECT(replies, get_query_reply_str(reply), counters->reply[reply]);
+	JSON_ADD_ITEM_TO_OBJECT(queries, "replies", replies);
+
+	// Count clients that have been active within the most recent 24 hours
+	unsigned int activeclients = 0;
+	for(int clientID=0; clientID < counters->clients; clientID++)
+	{
+		// Get client pointer
+		const clientsData* client = getClient(clientID, true);
+		if(client == NULL)
+			continue;
+
+		if(client->count > 0)
+			activeclients++;
+	}
+
+	cJSON *clients = JSON_NEW_OBJECT();
+	JSON_ADD_NUMBER_TO_OBJECT(clients, "active", activeclients);
+	JSON_ADD_NUMBER_TO_OBJECT(clients, "total", counters->clients);
+
+	cJSON *gravity = JSON_NEW_OBJECT();
+	JSON_ADD_NUMBER_TO_OBJECT(gravity, "domains_being_blocked", counters->database.gravity);
+
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_ADD_ITEM_TO_OBJECT(json, "queries", queries);
+	JSON_ADD_ITEM_TO_OBJECT(json, "clients", clients);
+	JSON_ADD_ITEM_TO_OBJECT(json, "gravity", gravity);
+	JSON_SEND_OBJECT_UNLOCK(json);
+}
+
+int api_stats_top_domains(struct ftl_conn *api)
+{
+	// Exit before processing any data if requested via config setting
+	if(config.misc.privacylevel.v.privacy_level >= PRIVACY_HIDE_DOMAINS)
+	{
+		log_debug(DEBUG_API, "Not returning top domains: Privacy level is set to %i",
+		          config.misc.privacylevel.v.privacy_level);
+
+		// Minimum structure is
+		// {"top_domains":[]}
+		cJSON *json = JSON_NEW_OBJECT();
+		cJSON *top_domains = JSON_NEW_ARRAY();
+		JSON_ADD_ITEM_TO_OBJECT(json, "top_domains", top_domains);
+		JSON_SEND_OBJECT(json);
+	}
+
+	// Lock shared memory
+	lock_shm();
+
+	// Allocate memory
+	const int domains = counters->domains;
+	int *temparray = calloc(2*domains, sizeof(int));
+	if(temparray == NULL)
+	{
+		log_err("Memory allocation failed in %s()", __FUNCTION__);
+		return 0;
+	}
+
+	bool blocked = false; // Can be overwritten by query string
+	int count = 10;
+	// /api/stats/top_domains?blocked=true
+	if(api->request->query_string != NULL)
+	{
+		// Should blocked domains be shown?
+		get_bool_var(api->request->query_string, "blocked", &blocked);
+
+		// Does the user request a non-default number of replies?
+		// Note: We do not accept zero query requests here
+		get_int_var(api->request->query_string, "count", &count);
+	}
+
+	unsigned int added_domains = 0u;
+	for(int domainID = 0; domainID < domains; domainID++)
+	{
+		// Get domain pointer
+		const domainsData* domain = getDomain(domainID, true);
+		if(domain == NULL)
+			continue;
+
+		// Add domain ID
+		temparray[2*added_domains + 0] = domainID;
+
+		// Use either blocked or total count based on request string
+		temparray[2*added_domains + 1] = blocked ? domain->blockedcount : domain->count - domain->blockedcount;
+
+		added_domains++;
+	}
+
+	// Sort temporary array
+	qsort(temparray, added_domains, sizeof(int[2]), cmpdesc);
+
+	// Get filter
+	const char* log_show = read_setupVarsconf("API_QUERY_LOG_SHOW");
+	bool showpermitted = true, showblocked = true;
+	if(log_show != NULL)
+	{
+		if((strcmp(log_show, "permittedonly")) == 0)
+			showblocked = false;
+		else if((strcmp(log_show, "blockedonly")) == 0)
+			showpermitted = false;
+		else if((strcmp(log_show, "nothing")) == 0)
+		{
+			showpermitted = false;
+			showblocked = false;
+		}
+	}
+	clearSetupVarsArray();
+
+	// Get domains which the user doesn't want to see
+	regex_t *regex_domains = NULL;
+	unsigned int N_regex_domains = 0;
+	compile_filter_regex(api, "webserver.api.excludeDomains",
+	                     config.webserver.api.excludeDomains.v.json,
+	                     &regex_domains, &N_regex_domains);
+
+	int n = 0;
+	cJSON *top_domains = JSON_NEW_ARRAY();
+	for(unsigned int i = 0; i < added_domains; i++)
+	{
+		// Get sorted index
+		const int domainID = temparray[2*i + 0];
+		// Get domain pointer
+		const domainsData* domain = getDomain(domainID, true);
+		if(domain == NULL)
+			continue;
+
+		// Get domain name
+		const char *domain_name = getstr(domain->domainpos);
+
+		// Hidden domain, probably due to privacy level. Skip this in the top lists
+		if(strcmp(domain_name, HIDDEN_DOMAIN) == 0)
+			continue;
+
+		// Skip this client if there is a filter on it
+		bool skip_domain = false;
+		if(N_regex_domains > 0)
+		{
+			// Iterate over all regex filters
+			for(unsigned int j = 0; j < N_regex_domains; j++)
+			{
+				// Check if the domain matches the regex
+				if(regexec(&regex_domains[j], domain_name, 0, NULL, 0) == 0)
+				{
+					// Domain matches
+					skip_domain = true;
+					break;
+				}
+			}
+		}
+
+		if(skip_domain)
+			continue;
+
+		int domain_count = -1;
+		if(blocked && showblocked && domain->blockedcount > 0)
+		{
+			domain_count = domain->blockedcount;
+			n++;
+		}
+		else if(!blocked && showpermitted && (domain->count - domain->blockedcount) > 0)
+		{
+			domain_count = domain->count - domain->blockedcount;
+			n++;
+		}
+		if(domain_count > -1)
+		{
+			cJSON *domain_item = JSON_NEW_OBJECT();
+			JSON_REF_STR_IN_OBJECT(domain_item, "domain", domain_name);
+			JSON_ADD_NUMBER_TO_OBJECT(domain_item, "count", domain_count);
+			JSON_ADD_ITEM_TO_ARRAY(top_domains, domain_item);
+		}
+
+		// Only count entries that are actually sent and return when we have send enough data
+		if(n >= count)
+			break;
+	}
+	free(temparray);
+
+	// Free regexes
+	if(N_regex_domains > 0)
+	{
+		// Free individual regexes
+		for(unsigned int i = 0; i < N_regex_domains; i++)
+			regfree(&regex_domains[i]);
+
+		// Free array of regex pointers
+		free(regex_domains);
+	}
+
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_ADD_ITEM_TO_OBJECT(json, "domains", top_domains);
+
+	const int blocked_count = get_blocked_count();
+	JSON_ADD_NUMBER_TO_OBJECT(json, "total_queries", counters->queries);
+	JSON_ADD_NUMBER_TO_OBJECT(json, "blocked_queries", blocked_count);
+
+	JSON_SEND_OBJECT_UNLOCK(json);
+}
+
+int api_stats_top_clients(struct ftl_conn *api)
+{
+	int count = 10;
+
+	// Exit before processing any data if requested via config setting
+	if(config.misc.privacylevel.v.privacy_level >= PRIVACY_HIDE_DOMAINS_CLIENTS)
+	{
+		log_debug(DEBUG_API, "Not returning top clients: Privacy level is set to %i",
+		          config.misc.privacylevel.v.privacy_level);
+
+		// Minimum structure is
+		// {"top_clients":[]}
+		cJSON *json = JSON_NEW_OBJECT();
+		cJSON *top_clients = JSON_NEW_ARRAY();
+		JSON_ADD_ITEM_TO_OBJECT(json, "top_clients", top_clients);
+		JSON_SEND_OBJECT(json);
+	}
+
+	bool blocked = false; // /api/stats/top_clients?blocked=true
+	if(api->request->query_string != NULL)
+	{
+		// Should blocked clients be shown?
+		get_bool_var(api->request->query_string, "blocked", &blocked);
+
+		// Does the user request a non-default number of replies?
+		// Note: We do not accept zero query requests here
+		get_int_var(api->request->query_string, "count", &count);
+	}
+
+	// Lock shared memory
+	lock_shm();
+
+	int clients = counters->clients;
+	int *temparray = calloc(2*clients, sizeof(int));
+	if(temparray == NULL)
+	{
+		log_err("Memory allocation failed in api_stats_top_clients()");
+		return 0;
+	}
+
+	unsigned int added_clients = 0;
+	for(int clientID = 0; clientID < clients; clientID++)
+	{
+		// Get client pointer
+		const clientsData* client = getClient(clientID, true);
+
+		// Skip invalid clients and also those managed by alias clients
+		if(client == NULL || (!client->flags.aliasclient && client->aliasclient_id >= 0))
+			continue;
+
+		temparray[2*added_clients + 0] = clientID;
+		// Use either blocked or total count based on request string
+		temparray[2*added_clients + 1] = blocked ? client->blockedcount : client->count;
+
+		added_clients++;
+	}
+
+	// Sort temporary array
+	qsort(temparray, added_clients, sizeof(int[2]), cmpdesc);
+
+	// Get clients which the user doesn't want to see
+	regex_t *regex_clients = NULL;
+	unsigned int N_regex_clients = 0;
+	compile_filter_regex(api, "webserver.api.excludeClients",
+	                     config.webserver.api.excludeClients.v.json,
+	                     &regex_clients, &N_regex_clients);
+
+	int n = 0;
+	cJSON *top_clients = JSON_NEW_ARRAY();
+	for(unsigned int i = 0; i < added_clients; i++)
+	{
+		// Get sorted indices and counter values (may be either total or blocked count)
+		const int clientID = temparray[2*i + 0];
+		const int client_count = temparray[2*i + 1];
+		// Get client pointer
+		const clientsData* client = getClient(clientID, true);
+		if(client == NULL)
+			continue;
+
+		// Get IP and host name of client
+		const char *client_ip = getstr(client->ippos);
+		const char *client_name = getstr(client->namepos);
+
+		// Hidden client, probably due to privacy level. Skip this in the top lists
+		if(strcmp(client_ip, HIDDEN_CLIENT) == 0)
+			continue;
+
+		// Skip this client if there is a filter on it
+		bool skip_client = false;
+		if(N_regex_clients > 0)
+		{
+			// Iterate over all regex filters
+			for(unsigned int j = 0; j < N_regex_clients; j++)
+			{
+				// Check if the domain matches the regex
+				if(regexec(&regex_clients[j], client_ip, 0, NULL, 0) == 0)
+				{
+					// Client IP matches
+					skip_client = true;
+					break;
+				}
+				else if(client_name != NULL && regexec(&regex_clients[j], client_name, 0, NULL, 0) == 0)
+				{
+					// Client name matches
+					skip_client = true;
+					break;
+				}
+			}
+		}
+
+		if(skip_client)
+			continue;
+
+		// Return this client if the client made at least one query
+		// within the most recent 24 hours
+		if(client_count > 0)
+		{
+			cJSON *client_item = JSON_NEW_OBJECT();
+			JSON_REF_STR_IN_OBJECT(client_item, "name", client_name);
+			JSON_REF_STR_IN_OBJECT(client_item, "ip", client_ip);
+			JSON_ADD_NUMBER_TO_OBJECT(client_item, "count", client_count);
+			JSON_ADD_ITEM_TO_ARRAY(top_clients, client_item);
+			n++;
+		}
+
+		if(n == count)
+			break;
+	}
+	// Free temporary array
+	free(temparray);
+
+	// Free regexes
+	if(N_regex_clients > 0)
+	{
+		// Free individual regexes
+		for(unsigned int i = 0; i < N_regex_clients; i++)
+			regfree(&regex_clients[i]);
+
+		// Free array of regex pointers
+		free(regex_clients);
+	}
+
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_ADD_ITEM_TO_OBJECT(json, "clients", top_clients);
+
+	const int blocked_count = get_blocked_count();
+	JSON_ADD_NUMBER_TO_OBJECT(json, "blocked_queries", blocked_count);
+	JSON_ADD_NUMBER_TO_OBJECT(json, "total_queries", counters->queries);
+	JSON_SEND_OBJECT_UNLOCK(json);
+}
+
+
+int api_stats_upstreams(struct ftl_conn *api)
+{
+	unsigned int totalcount = 0;
+	const int upstreams = counters->upstreams;
+	int *temparray = calloc(2*upstreams, sizeof(int));
+	if(temparray == NULL)
+	{
+		log_err("Memory allocation failed in api_stats_upstreams()");
+		return 0;
+	}
+
+	// Lock shared memory
+	lock_shm();
+
+	unsigned int added_upstreams = 0;
+	for(int upstreamID = 0; upstreamID < upstreams; upstreamID++)
+	{
+		// Get upstream pointer
+		const upstreamsData* upstream = getUpstream(upstreamID, true);
+		if(upstream == NULL)
+			continue;
+
+		temparray[2*added_upstreams + 0] = upstreamID;
+		temparray[2*added_upstreams + 1] = upstream->count;
+		totalcount += upstream->count;
+
+		added_upstreams++;
+	}
+
+	// Sort temporary array in descending order
+	qsort(temparray, upstreams, sizeof(int[2]), cmpdesc);
+
+	// Loop over available forward destinations
+	cJSON *top_upstreams = JSON_NEW_ARRAY();
+	for(int i = -2; i < (int)added_upstreams; i++)
+	{
+		int count = 0;
+		const char* ip, *name;
+		int port = -1;
+		double responsetime = 0.0, uncertainty = 0.0;
+
+		if(i == -2)
+		{
+			// Blocked queries (local lists)
+			ip = "blocklist";
+			name = ip;
+			count = get_blocked_count();
+		}
+		else if(i == -1)
+		{
+			// Local cache
+			ip = "cache";
+			name = ip;
+			count = get_cached_count();
+		}
+		else
+		{
+			// Regular upstream destination
+			// Get sorted indices
+			const int upstreamID = temparray[2*i + 0];
+
+			// Get upstream pointer
+			const upstreamsData *upstream = getUpstream(upstreamID, true);
+			if(upstream == NULL)
+				continue;
+
+			// Get IP and host name of upstream destination if available
+			ip = getstr(upstream->ippos);
+			name = getstr(upstream->namepos);
+			port = upstream->port;
+
+			// Get percentage
+			count = upstream->count;
+
+			// Compute average response time and uncertainty (unit: seconds)
+			if(upstream->responses > 0)
+			{
+				// Simple average of the response times
+				responsetime = upstream->rtime / upstream->responses;
+			}
+			if(upstream->responses > 1)
+			{
+				// The actual value will be somewhere in a neighborhood around the mean value.
+				// This neighborhood of values is the uncertainty in the mean.
+				uncertainty = sqrt(upstream->rtuncertainty / upstream->responses / (upstream->responses-1));
+			}
+		}
+
+		// Send data:
+		// - always if i < 0 (special upstreams: blocklist and cache)
+		// - only if there are any queries for all others (i > 0)
+		if(count > 0 || i < 0)
+		{
+			cJSON *upstream = JSON_NEW_OBJECT();
+			JSON_REF_STR_IN_OBJECT(upstream, "ip", ip);
+			JSON_REF_STR_IN_OBJECT(upstream, "name", name);
+			JSON_ADD_NUMBER_TO_OBJECT(upstream, "port", port);
+			JSON_ADD_NUMBER_TO_OBJECT(upstream, "count", count);
+			cJSON *statistics = JSON_NEW_OBJECT();
+			JSON_ADD_NUMBER_TO_OBJECT(statistics, "response", responsetime);
+			JSON_ADD_NUMBER_TO_OBJECT(statistics, "variance", uncertainty);
+			JSON_ADD_ITEM_TO_OBJECT(upstream, "statistics", statistics);
+			JSON_ADD_ITEM_TO_ARRAY(top_upstreams, upstream);
+		}
+	}
+
+	// Free temporary array
+	free(temparray);
+
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_ADD_ITEM_TO_OBJECT(json, "upstreams", top_upstreams);
+	const int forwarded_count = get_forwarded_count();
+	JSON_ADD_NUMBER_TO_OBJECT(json, "forwarded_queries", forwarded_count);
+	JSON_ADD_NUMBER_TO_OBJECT(json, "total_queries", counters->queries);
+	JSON_SEND_OBJECT_UNLOCK(json);
+}
+
+int api_stats_query_types(struct ftl_conn *api)
+{
+	lock_shm();
+
+	cJSON *types = JSON_NEW_OBJECT();
+	int ret = get_query_types_obj(api, types);
+	if(ret != 0)
+	{
+		unlock_shm();
+		return ret;
+	}
+
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_ADD_ITEM_TO_OBJECT(json, "types", types);
+
+	// Send response
+	JSON_SEND_OBJECT_UNLOCK(json);
+}
+
+int api_stats_recentblocked(struct ftl_conn *api)
+{
+	// Exit before processing any data if requested via config setting
+	if(config.misc.privacylevel.v.privacy_level >= PRIVACY_HIDE_DOMAINS)
+	{
+		// Minimum structure is
+		// {"blocked":[]}
+		cJSON *json = JSON_NEW_OBJECT();
+		cJSON *blocked = JSON_NEW_ARRAY();
+		JSON_ADD_ITEM_TO_OBJECT(json, "blocked", blocked);
+		JSON_SEND_OBJECT(json);
+	}
+
+	unsigned int count = 1;
+	if(api->request->query_string != NULL)
+	{
+		// Does the user request a non-default number of replies?
+		// Note: We do not accept zero query requests here
+		get_uint_var(api->request->query_string, "count", &count);
+	}
+
+	// Lock shared memory
+	lock_shm();
+
+	// Find most recently blocked query
+	unsigned int found = 0;
+	cJSON *blocked = JSON_NEW_ARRAY();
+	for(int queryID = counters->queries - 1; queryID > 0 ; queryID--)
+	{
+		const queriesData* query = getQuery(queryID, true);
+		if(query == NULL)
+			continue;
+
+		if(query->flags.blocked)
+		{
+			// Ask subroutine for domain. It may return "hidden" depending on
+			// the privacy settings at the time the query was made
+			const char *domain = getDomainString(query);
+			if(domain == NULL)
+				continue;
+
+			JSON_REF_STR_IN_ARRAY(blocked, domain);
+
+			// Only count when added successfully
+			found++;
+		}
+
+		if(found >= count)
+			break;
+	}
+
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_ADD_ITEM_TO_OBJECT(json, "blocked", blocked);
+	JSON_SEND_OBJECT_UNLOCK(json);
+}

src/api/stats_database.c

@@ -0,0 +1,841 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2019 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  API database statistics implementation
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#include "../FTL.h"
+#include "../webserver/http-common.h"
+#include "../webserver/json_macros.h"
+#include "api.h"
+// querytypes[]
+#include "../datastructure.h"
+// logging routines
+#include "log.h"
+// db
+#include "../database/common.h"
+
+// SQL Query type filters for the database
+#define FILTER_STATUS_NOT_BLOCKED "status IN (0,2,3,12,13,14,17)"
+#define FILTER_STATUS_BLOCKED "status NOT IN (0,2,3,12,13,14,17)"
+
+int api_history_database(struct ftl_conn *api)
+{
+	double from = 0, until = 0;
+	const int interval = 600;
+	if(api->request->query_string != NULL)
+	{
+		get_double_var(api->request->query_string, "from", &from);
+		get_double_var(api->request->query_string, "until", &until);
+	}
+
+	// Check if we received the required information
+	if(from < 1.0 || until < 1.0)
+	{
+		return send_json_error(api, 400,
+		                       "bad_request",
+		                       "You need to specify both \"from\" and \"until\" in the request.",
+		                       NULL);
+	}
+
+	// Open the database
+	sqlite3 *db = dbopen(false, false);
+	if(db == NULL)
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to open long-term database",
+		                       NULL);
+
+	// Build SQL string
+	const char *querystr = "SELECT (timestamp/:interval)*:interval interval,status,COUNT(*) FROM query_storage "
+	                       "WHERE (status != 0) AND timestamp >= :from AND timestamp <= :until "
+	                       "GROUP by interval,status ORDER by interval";
+
+
+	// Prepare SQLite statement
+	sqlite3_stmt *stmt;
+	int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL);
+	if( rc != SQLITE_OK ){
+		log_err("api_stats_database_history() - SQL error prepare (%i): %s",
+		        rc, sqlite3_errstr(rc));
+		return false;
+	}
+
+	// Bind interval to prepared statement
+	if((rc = sqlite3_bind_int(stmt, 1, interval)) != SQLITE_OK)
+	{
+		log_err("api_stats_database_history(): Failed to bind interval (error %d) - %s",
+		        rc, sqlite3_errstr(rc));
+		sqlite3_reset(stmt);
+		sqlite3_finalize(stmt);
+		dbclose(&db);
+
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to bind interval",
+		                       NULL);
+	}
+
+	// Bind from to prepared statement
+	if((rc = sqlite3_bind_double(stmt, 2, from)) != SQLITE_OK)
+	{
+		log_err("api_stats_database_history(): Failed to bind from (error %d) - %s",
+		        rc, sqlite3_errstr(rc));
+		sqlite3_reset(stmt);
+		sqlite3_finalize(stmt);
+		dbclose(&db);
+
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to bind from",
+		                       NULL);
+	}
+
+	// Bind until to prepared statement
+	if((rc = sqlite3_bind_double(stmt, 3, until)) != SQLITE_OK)
+	{
+		log_err("api_stats_database_history(): Failed to bind until (error %d) - %s",
+		        rc, sqlite3_errstr(rc));
+		sqlite3_reset(stmt);
+		sqlite3_finalize(stmt);
+		dbclose(&db);
+
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to bind until",
+		                       NULL);
+	}
+
+	// Loop over returned data and accumulate results
+	cJSON *history = JSON_NEW_ARRAY();
+	cJSON *item = NULL;
+	unsigned int previous_timeslot = 0u, blocked = 0u, total = 0u, cached = 0u;
+	while((rc = sqlite3_step(stmt)) == SQLITE_ROW)
+	{
+		// Get timestamp and derive timeslot from it
+		const unsigned int timestamp = sqlite3_column_int(stmt, 0);
+		const unsigned int timeslot = timestamp - timestamp % interval;
+		// Begin new array item for each new timeslot
+		if(timeslot != previous_timeslot)
+		{
+			previous_timeslot = timeslot;
+			if(item != NULL)
+			{
+				// Add and reset total counter
+				JSON_ADD_NUMBER_TO_OBJECT(item, "total", total);
+				total = 0;
+				// Add and reset totacachedl counter
+				JSON_ADD_NUMBER_TO_OBJECT(item, "cached", cached);
+				cached = 0;
+				// Add and reset blocked counter
+				JSON_ADD_NUMBER_TO_OBJECT(item, "blocked", blocked);
+				blocked = 0;
+				JSON_ADD_ITEM_TO_ARRAY(history, item);
+			}
+
+			item = JSON_NEW_OBJECT();
+			JSON_ADD_NUMBER_TO_OBJECT(item, "timestamp", previous_timeslot);
+		}
+
+		const int status = sqlite3_column_int(stmt, 1);
+		const int count = sqlite3_column_int(stmt, 2);
+		// Always add to total count
+		total += count;
+
+		// Add to blocked / cached count if applicable
+		if(is_blocked(status))
+			blocked += count;
+		else if(is_cached(status))
+			cached += count;
+	}
+
+	// Append final timeslot at the end if applicable
+	if(total > 0 && item != NULL)
+	{
+		// Add total counter
+		JSON_ADD_NUMBER_TO_OBJECT(item, "total", total);
+		// Add cached counter
+		JSON_ADD_NUMBER_TO_OBJECT(item, "cached", cached);
+		// Add blocked counter
+		JSON_ADD_NUMBER_TO_OBJECT(item, "blocked", blocked);
+		JSON_ADD_ITEM_TO_ARRAY(history, item);
+	}
+
+	// Finalize statement and close (= unlock) database connection
+	sqlite3_finalize(stmt);
+	dbclose(&db);
+
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_ADD_ITEM_TO_OBJECT(json, "history", history);
+	JSON_SEND_OBJECT(json);
+}
+
+int api_stats_database_top_items(struct ftl_conn *api)
+{
+	unsigned int count = 10;
+	double from = 0.0, until = 0.0;
+
+	// Get options from API struct
+	bool blocked = false; // Can be overwritten by query string
+	const bool domains = api->opts.flags & API_DOMAINS;
+
+	// Get parameters from query string
+	if(api->request->query_string != NULL)
+	{
+		// Get time interval from query string
+		get_double_var(api->request->query_string, "from", &from);
+		get_double_var(api->request->query_string, "until", &until);
+
+		// Get blocked queries not only for .../top_blocked
+		// but also for .../top_domains?blocked=true
+		// Note: this may overwrite the blocked property from the URL
+		get_bool_var(api->request->query_string, "blocked", &blocked);
+
+		// Does the user request a non-default number of replies?
+		// Note: We do not accept zero query requests here
+		get_uint_var(api->request->query_string, "count", &count);
+	}
+
+	// Check if we received the required information
+	if(from < 1.0 || until < 1.0)
+	{
+		return send_json_error(api, 400,
+		                       "bad_request",
+		                       "You need to specify both \"from\" and \"until\" in the request.",
+		                       NULL);
+	}
+
+	// Open the database
+	sqlite3 *db = dbopen(false, false);
+	if(db == NULL)
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to open long-term database",
+		                       NULL);
+
+	// Build SQL string
+	const char *querystr, *count_total_str, *count_blocked_str;
+	if(domains)
+	{
+		if(blocked)
+		{
+			// Get domains and count of queries (blocked)
+			querystr = "SELECT COUNT(*),d.domain AS cnt FROM query_storage q "
+			           "JOIN domain_by_id d ON d.id = q.domain "
+			           "WHERE timestamp >= :from AND timestamp <= :until "
+			           "AND " FILTER_STATUS_BLOCKED " "
+			           "GROUP by q.domain";
+		}
+		else
+		{
+			// Get domains and count of queries (not blocked)
+			querystr = "SELECT COUNT(*),d.domain AS cnt FROM query_storage q "
+			           "JOIN domain_by_id d ON d.id = q.domain "
+			           "WHERE timestamp >= :from AND timestamp <= :until "
+			           "AND " FILTER_STATUS_NOT_BLOCKED " "
+			           "GROUP by q.domain";
+		}
+
+		// Count total number of queries for domains
+		count_total_str = "SELECT COUNT(DISTINCT domain) FROM query_storage "
+		                  "WHERE timestamp >= :from AND timestamp <= :until";
+
+		// Count total number of blocked queries for domains
+		count_blocked_str = "SELECT COUNT(DISTINCT domain) FROM query_storage "
+		                    "WHERE timestamp >= :from AND timestamp <= :until "
+			            "AND " FILTER_STATUS_BLOCKED;
+	}
+	else
+	{
+		if(blocked)
+		{
+			// Get clients and count of queries (blocked)
+			querystr = "SELECT COUNT(*),c.ip,c.name AS cnt FROM query_storage q "
+			           "JOIN client_by_id c ON c.id = q.client"
+			           "WHERE timestamp >= :from AND timestamp <= :until "
+			           "AND " FILTER_STATUS_BLOCKED " "
+			           "GROUP by q.client";
+		}
+		else
+		{
+			// Get clients and count of queries (not blocked)
+			querystr = "SELECT COUNT(*),c.ip,c.name AS cnt FROM query_storage q "
+			           "JOIN client_by_id c ON c.id = q.client "
+			           "WHERE timestamp >= :from AND timestamp <= :until "
+			           "AND " FILTER_STATUS_NOT_BLOCKED " "
+			           "GROUP by q.client";
+		}
+
+		// Count total number of queries for clients
+		count_total_str = "SELECT COUNT(DISTINCT client) FROM query_storage "
+		                  "WHERE timestamp >= :from AND timestamp <= :until";
+
+		// Count number of blocked queries for clients
+		count_blocked_str = "SELECT COUNT(DISTINCT client) FROM query_storage "
+		                    "WHERE timestamp >= :from AND timestamp <= :until "
+			            "AND " FILTER_STATUS_BLOCKED;
+	}
+
+
+	// Prepare SQLite statement
+	sqlite3_stmt *stmt;
+	int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL);
+	if( rc != SQLITE_OK ){
+		log_err("api_stats_database_history() - SQL error prepare (%i): %s",
+		        rc, sqlite3_errstr(rc));
+
+		dbclose(&db);
+
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to prepare query string",
+		                       querystr);
+	}
+
+	// Bind from to prepared statement
+	if((rc = sqlite3_bind_double(stmt, 1, from)) != SQLITE_OK)
+	{
+		log_err("api_stats_database_history(): Failed to bind from (error %d) - %s",
+		        rc, sqlite3_errstr(rc));
+		sqlite3_reset(stmt);
+		sqlite3_finalize(stmt);
+		dbclose(&db);
+
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to bind from",
+		                       NULL);
+	}
+
+	// Bind until to prepared statement
+	if((rc = sqlite3_bind_double(stmt, 2, until)) != SQLITE_OK)
+	{
+		log_err("api_stats_database_history(): Failed to bind until (error %d) - %s",
+		        rc, sqlite3_errstr(rc));
+		sqlite3_reset(stmt);
+		sqlite3_finalize(stmt);
+		dbclose(&db);
+
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to bind until",
+		                       NULL);
+	}
+
+	// Loop over and accumulate results
+	cJSON *top_items = JSON_NEW_ARRAY();
+	unsigned int total = 0;
+	while((rc = sqlite3_step(stmt)) == SQLITE_ROW &&
+	       ++total < count)
+	{
+		// Get count
+		const int cnt = sqlite3_column_int(stmt, 0);
+		cJSON *item = JSON_NEW_OBJECT();
+		if(domains)
+		{
+			// Add domain to item
+			JSON_COPY_STR_TO_OBJECT(item, "domain", sqlite3_column_text(stmt, 1));
+		}
+		else
+		{
+			// Add client to item
+			JSON_COPY_STR_TO_OBJECT(item, "ip", sqlite3_column_text(stmt, 1));
+			JSON_COPY_STR_TO_OBJECT(item, "name", sqlite3_column_text(stmt, 2));
+		}
+		JSON_ADD_NUMBER_TO_OBJECT(item, "count", cnt);
+		JSON_ADD_ITEM_TO_ARRAY(top_items, item);
+	}
+
+	// Finalize statement and close (= unlock) database connection
+	sqlite3_finalize(stmt);
+
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_ADD_ITEM_TO_OBJECT(json, (domains ? "domains" : "clients"), top_items);
+	const int total_num = db_query_int_from_until(db, count_total_str, from, until);
+	const int blocked_num = db_query_int_from_until(db, count_blocked_str, from, until);
+	JSON_ADD_NUMBER_TO_OBJECT(json, "total_queries", total_num);
+	JSON_ADD_NUMBER_TO_OBJECT(json, "blocked_queries", blocked_num);
+
+	dbclose(&db);
+	JSON_SEND_OBJECT(json);
+}
+
+int api_stats_database_summary(struct ftl_conn *api)
+{
+	double from = 0, until = 0;
+	if(api->request->query_string != NULL)
+	{
+		get_double_var(api->request->query_string, "from", &from);
+		get_double_var(api->request->query_string, "until", &until);
+	}
+
+	// Check if we received the required information
+	if(from < 1.0 || until < 1.0)
+	{
+		return send_json_error(api, 400,
+		                       "bad_request",
+		                       "You need to specify both \"from\" and \"until\" in the request.",
+		                       NULL);
+	}
+
+	// Open the database
+	sqlite3 *db = dbopen(false, false);
+	if(db == NULL)
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to open long-term database",
+		                       NULL);
+
+	// Perform SQL queries
+	const char *querystr;
+	querystr = "SELECT COUNT(*) FROM query_storage "
+	           "WHERE timestamp >= :from AND timestamp <= :until";
+	const int sum_queries = db_query_int_from_until(db, querystr, from, until);
+
+	querystr = "SELECT COUNT(*) FROM query_storage "
+	           "WHERE timestamp >= :from AND timestamp <= :until "
+	           "AND " FILTER_STATUS_BLOCKED;
+	const int sum_blocked = db_query_int_from_until(db, querystr, from, until);
+
+	querystr = "SELECT COUNT(DISTINCT client) FROM query_storage "
+	           "WHERE timestamp >= :from AND timestamp <= :until";
+	const int total_clients = db_query_int_from_until(db, querystr, from, until);
+
+	// Calculate percentage of blocked queries, substituting 0.0 if there
+	// are no blocked queries
+	float percent_blocked = 0.0;
+	if(sum_queries > 0.0)
+		percent_blocked = 1e2f*sum_blocked/sum_queries;
+
+	if(sum_queries < 0 || sum_blocked < 0 || total_clients < 0)
+	{
+
+		// Close (= unlock) database connection
+		dbclose(&db);
+
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Internal server error",
+		                       NULL);
+	}
+
+	// Loop over and accumulate results
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_ADD_NUMBER_TO_OBJECT(json, "sum_queries", sum_queries);
+	JSON_ADD_NUMBER_TO_OBJECT(json, "sum_blocked", sum_blocked);
+	JSON_ADD_NUMBER_TO_OBJECT(json, "percent_blocked", percent_blocked);
+	JSON_ADD_NUMBER_TO_OBJECT(json, "total_clients", total_clients);
+
+	// Close (= unlock) database connection
+	dbclose(&db);
+
+	// Send JSON object
+	JSON_SEND_OBJECT(json);
+}
+
+int api_history_database_clients(struct ftl_conn *api)
+{
+	double from = 0, until = 0;
+	const int interval = 600;
+	if(api->request->query_string != NULL)
+	{
+		get_double_var(api->request->query_string, "from", &from);
+		get_double_var(api->request->query_string, "until", &until);
+	}
+
+	// Check if we received the required information
+	if(from < 1.0 || until < 1.0)
+	{
+		return send_json_error(api, 400,
+		                       "bad_request",
+		                       "You need to specify both \"from\" and \"until\" in the request.",
+		                       NULL);
+	}
+
+	// Open the database
+	sqlite3 *db = dbopen(false, false);
+	if(db == NULL)
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to open long-term database",
+		                       NULL);
+
+	const char *querystr = "SELECT DISTINCT(client),ip,name FROM query_storage "
+	                       "JOIN client_by_id ON client_by_id.id = client "
+	                       "WHERE timestamp >= :from AND timestamp <= :until "
+	                       "ORDER BY client DESC";
+
+	// Prepare SQLite statement
+	sqlite3_stmt *stmt;
+	int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL);
+	if( rc != SQLITE_OK ){
+		log_err("api_stats_database_clients() - SQL error prepare outer (%i): %s",
+		        rc, sqlite3_errstr(rc));
+
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to prepare outer statement",
+		                       NULL);
+	}
+
+	// Bind from to prepared statement
+	if((rc = sqlite3_bind_double(stmt, 1, from)) != SQLITE_OK)
+	{
+		log_err("api_stats_database_clients(): Failed to bind from (error %d) - %s",
+		        rc, sqlite3_errstr(rc));
+		sqlite3_reset(stmt);
+		sqlite3_finalize(stmt);
+		dbclose(&db);
+
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to bind from",
+		                       NULL);
+	}
+
+	// Bind until to prepared statement
+	if((rc = sqlite3_bind_double(stmt, 2, until)) != SQLITE_OK)
+	{
+		log_err("api_stats_database_clients(): Failed to bind until (error %d) - %s",
+		        rc, sqlite3_errstr(rc));
+		sqlite3_reset(stmt);
+		sqlite3_finalize(stmt);
+		dbclose(&db);
+
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to bind until",
+		                       NULL);
+	}
+
+	// Loop over clients and accumulate results
+	cJSON *clients = JSON_NEW_OBJECT();
+	unsigned int num_clients = 0;
+	while((rc = sqlite3_step(stmt)) == SQLITE_ROW)
+	{
+		cJSON *item = JSON_NEW_OBJECT();
+		JSON_COPY_STR_TO_OBJECT(item, "name", sqlite3_column_text(stmt, 2));
+		JSON_ADD_ITEM_TO_OBJECT(clients, (const char*)sqlite3_column_text(stmt, 1), item);
+		num_clients++;
+	}
+	sqlite3_finalize(stmt);
+
+	// Build SQL string
+	querystr = "SELECT (timestamp/:interval)*:interval interval,client,COUNT(*) FROM query_storage "
+	           "WHERE timestamp >= :from AND timestamp <= :until "
+	           "GROUP BY interval,client ORDER BY interval DESC, client DESC";
+
+	// Prepare SQLite statement
+	rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL);
+	if( rc != SQLITE_OK ){
+		log_err("api_stats_database_clients() - SQL error prepare (%i): %s",
+		   rc, sqlite3_errstr(rc));
+
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to prepare inner statement",
+		                       NULL);
+	}
+
+	// Bind interval to prepared statement
+	if((rc = sqlite3_bind_int(stmt, 1, interval)) != SQLITE_OK)
+	{
+		log_err("api_stats_database_clients(): Failed to bind interval (error %d) - %s",
+		        rc, sqlite3_errstr(rc));
+		sqlite3_reset(stmt);
+		sqlite3_finalize(stmt);
+		dbclose(&db);
+
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to bind interval",
+		                       NULL);
+	}
+
+	// Bind from to prepared statement
+	if((rc = sqlite3_bind_int(stmt, 2, from)) != SQLITE_OK)
+	{
+		log_err("api_stats_database_clients(): Failed to bind from (error %d) - %s",
+		        rc, sqlite3_errstr(rc));
+		sqlite3_reset(stmt);
+		sqlite3_finalize(stmt);
+		dbclose(&db);
+
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to bind from",
+		                       NULL);
+	}
+
+	// Bind until to prepared statement
+	if((rc = sqlite3_bind_int(stmt, 3, until)) != SQLITE_OK)
+	{
+		log_err("api_stats_database_clients(): Failed to bind until (error %d) - %s",
+		        rc, sqlite3_errstr(rc));
+		sqlite3_reset(stmt);
+		sqlite3_finalize(stmt);
+		dbclose(&db);
+
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to bind until",
+		                       NULL);
+	}
+
+	cJSON *item = NULL;
+	cJSON *data = NULL;
+	unsigned int previous_timeslot = 0u;
+	cJSON *over_time = JSON_NEW_ARRAY();
+	while((rc = sqlite3_step(stmt)) == SQLITE_ROW)
+	{
+		// Get timestamp and derive timeslot from it
+		const unsigned int timestamp = sqlite3_column_int(stmt, 0);
+		const unsigned int timeslot = timestamp - timestamp % interval;
+		// Begin new array item for each new timeslot
+		if(timeslot != previous_timeslot)
+		{
+			previous_timeslot = timeslot;
+			if(item != NULL && data != NULL)
+			{
+				JSON_ADD_ITEM_TO_OBJECT(item, "data", data);
+				JSON_ADD_ITEM_TO_ARRAY(over_time, item);
+			}
+
+			item = JSON_NEW_OBJECT();
+			data = JSON_NEW_OBJECT();
+
+			JSON_ADD_NUMBER_TO_OBJECT(item, "timestamp", previous_timeslot);
+		}
+
+		const char *client = (char*)sqlite3_column_text(stmt, 1);
+		const int count = sqlite3_column_int(stmt, 2);
+
+		JSON_ADD_NUMBER_TO_OBJECT(data, client, count);
+	}
+
+	// Append final timeslot at the end if applicable
+	if(item != NULL && data != NULL)
+	{
+		JSON_ADD_ITEM_TO_OBJECT(item, "data", data);
+		JSON_ADD_ITEM_TO_ARRAY(over_time, item);
+	}
+
+	// Finalize statement and close (= unlock) database connection
+	sqlite3_finalize(stmt);
+	dbclose(&db);
+
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_ADD_ITEM_TO_OBJECT(json, "history", over_time);
+	JSON_ADD_ITEM_TO_OBJECT(json, "clients", clients);
+	JSON_SEND_OBJECT(json);
+}
+
+int api_stats_database_query_types(struct ftl_conn *api)
+{
+	double from = 0, until = 0;
+	if(api->request->query_string != NULL)
+	{
+		get_double_var(api->request->query_string, "from", &from);
+		get_double_var(api->request->query_string, "until", &until);
+	}
+
+	// Check if we received the required information
+	if(from < 1.0 || until < 1.0)
+	{
+		return send_json_error(api, 400,
+		                       "bad_request",
+		                       "You need to specify both \"from\" and \"until\" in the request.",
+		                       NULL);
+	}
+
+	// Open the database
+	sqlite3 *db = dbopen(false, false);
+	if(db == NULL)
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to open long-term database",
+		                       NULL);
+
+	// Perform SQL queries
+	cJSON *types = JSON_NEW_OBJECT();
+	for(int i = TYPE_A; i < TYPE_MAX; i++)
+	{
+		const char *querystr = "SELECT COUNT(*) FROM query_storage "
+		                       "WHERE timestamp >= :from AND timestamp <= :until "
+		                       "AND type = :type";
+		// Add 1 as type is stored one-based in the database for historical reasons
+		int count = db_query_int_from_until_type(db, querystr, from, until, i+1);
+		JSON_ADD_NUMBER_TO_OBJECT(types, get_query_type_str(i, NULL, NULL), count);
+	}
+
+	// Close (= unlock) database connection
+	dbclose(&db);
+
+	// Send JSON object
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_ADD_ITEM_TO_OBJECT(json, "types", types);
+	JSON_SEND_OBJECT(json);
+}
+
+
+int api_stats_database_upstreams(struct ftl_conn *api)
+{
+	double from = 0, until = 0;
+	if(api->request->query_string != NULL)
+	{
+		get_double_var(api->request->query_string, "from", &from);
+		get_double_var(api->request->query_string, "until", &until);
+	}
+
+	// Check if we received the required information
+	if(from < 1.0 || until < 1.0)
+	{
+		return send_json_error(api, 400,
+		                       "bad_request",
+		                       "You need to specify both \"from\" and \"until\" in the request.",
+		                       NULL);
+	}
+
+	// Open the database
+	sqlite3 *db = dbopen(false, false);
+	if(db == NULL)
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to open long-term database",
+		                       NULL);
+
+	// Perform simple SQL queries
+	unsigned int sum_queries = 0;
+	const char *querystr;
+	querystr = "SELECT COUNT(*) FROM query_storage "
+	           "WHERE timestamp >= :from AND timestamp <= :until "
+	           "AND status = 3";
+	int cached_queries = db_query_int_from_until(db, querystr, from, until);
+	sum_queries += cached_queries;
+
+	querystr = "SELECT COUNT(*) FROM query_storage "
+	           "WHERE timestamp >= :from AND timestamp <= :until "
+		   "AND status != 0 AND status != 2 AND status != 3";
+	int blocked_queries = db_query_int_from_until(db, querystr, from, until);
+	sum_queries += blocked_queries;
+
+	querystr = "SELECT forward,COUNT(*) FROM query_storage "
+	           "WHERE timestamp >= :from AND timestamp <= :until "
+		   "AND forward IS NOT NULL "
+	           "GROUP BY forward ORDER BY forward";
+
+	// Prepare SQLite statement
+	sqlite3_stmt *stmt;
+	int rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL);
+	if( rc != SQLITE_OK ){
+		log_err("api_stats_database_clients() - SQL error prepare (%i): %s",
+		        rc, sqlite3_errstr(rc));
+
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to prepare statement",
+		                       NULL);
+	}
+
+	// Bind from to prepared statement
+	if((rc = sqlite3_bind_double(stmt, 1, from)) != SQLITE_OK)
+	{
+		log_err("api_stats_database_clients(): Failed to bind from (error %d) - %s",
+		        rc, sqlite3_errstr(rc));
+		sqlite3_reset(stmt);
+		sqlite3_finalize(stmt);
+		dbclose(&db);
+
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to bind from",
+		                       NULL);
+	}
+
+	// Bind until to prepared statement
+	if((rc = sqlite3_bind_double(stmt, 2, until)) != SQLITE_OK)
+	{
+		log_err("api_stats_database_clients(): Failed to bind until (error %d) - %s",
+		        rc, sqlite3_errstr(rc));
+		sqlite3_reset(stmt);
+		sqlite3_finalize(stmt);
+		dbclose(&db);
+
+		return send_json_error(api, 500,
+		                       "internal_error",
+		                       "Failed to bind until",
+		                       NULL);
+	}
+
+	// Loop over clients and accumulate results
+	cJSON *upstreams = JSON_NEW_ARRAY();
+	int forwarded_queries = 0;
+	while((rc = sqlite3_step(stmt)) == SQLITE_ROW)
+	{
+		const char *upstream = (char*)sqlite3_column_text(stmt, 0);
+		const int count = sqlite3_column_int(stmt, 1);
+
+		cJSON *item = JSON_NEW_OBJECT();
+		unsigned int port = -1;
+		char buffer[512] =  { 0 };
+		if(sscanf(upstream, "%511[^#]#%u", buffer, &port) == 2)
+		{
+			buffer[sizeof(buffer)-1] = '\0';
+			JSON_COPY_STR_TO_OBJECT(item, "ip", buffer);
+		}
+		else
+			JSON_COPY_STR_TO_OBJECT(item, "ip", upstream);
+		JSON_REF_STR_IN_OBJECT(item, "name", "");
+		JSON_ADD_NUMBER_TO_OBJECT(item, "port", port);
+		JSON_ADD_NUMBER_TO_OBJECT(item, "count", count);
+
+		cJSON *statistics = JSON_NEW_OBJECT();
+		JSON_ADD_NUMBER_TO_OBJECT(statistics, "response", 0);
+		JSON_ADD_NUMBER_TO_OBJECT(statistics, "variance", 0);
+		JSON_ADD_ITEM_TO_OBJECT(item, "statistics", statistics);
+
+		JSON_ADD_ITEM_TO_ARRAY(upstreams, item);
+		forwarded_queries += count;
+	}
+	sqlite3_finalize(stmt);
+
+	// Add number of forwarded queries to total query count
+	sum_queries += forwarded_queries;
+
+	// Add cache and blocklist as upstreams
+	cJSON *cached = JSON_NEW_OBJECT();
+	JSON_REF_STR_IN_OBJECT(cached, "ip", "cache");
+	JSON_REF_STR_IN_OBJECT(cached, "name", "cache");
+	JSON_ADD_NUMBER_TO_OBJECT(cached, "port", -1);
+	JSON_ADD_NUMBER_TO_OBJECT(cached, "count", cached_queries);
+	cJSON *statistics = JSON_NEW_OBJECT();
+	JSON_ADD_NUMBER_TO_OBJECT(statistics, "response", 0);
+	JSON_ADD_NUMBER_TO_OBJECT(statistics, "variance", 0);
+	JSON_ADD_ITEM_TO_OBJECT(cached, "statistics", statistics);
+	JSON_ADD_ITEM_TO_ARRAY(upstreams, cached);
+
+	cJSON *blocked = JSON_NEW_OBJECT();
+	JSON_REF_STR_IN_OBJECT(blocked, "ip", "blocklist");
+	JSON_REF_STR_IN_OBJECT(blocked, "name", "blocklist");
+	JSON_ADD_NUMBER_TO_OBJECT(blocked, "port", -1);
+	JSON_ADD_NUMBER_TO_OBJECT(blocked, "count", blocked_queries);
+	statistics = JSON_NEW_OBJECT();
+	JSON_ADD_NUMBER_TO_OBJECT(statistics, "response", 0);
+	JSON_ADD_NUMBER_TO_OBJECT(statistics, "variance", 0);
+	JSON_ADD_ITEM_TO_OBJECT(cached, "statistics", statistics);
+	JSON_ADD_ITEM_TO_ARRAY(upstreams, blocked);
+
+	// Close (= unlock) database connection
+	dbclose(&db);
+
+	// Send JSON object
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_ADD_ITEM_TO_OBJECT(json, "upstreams", upstreams);
+	JSON_ADD_NUMBER_TO_OBJECT(json, "forwarded_queries", forwarded_queries);
+	JSON_ADD_NUMBER_TO_OBJECT(json, "total_queries", sum_queries);
+	JSON_SEND_OBJECT(json);
+}

src/api/teleporter.c

@@ -0,0 +1,830 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2023 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  API Implementation /api/teleporter
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#include "FTL.h"
+#include "webserver/http-common.h"
+#include "webserver/json_macros.h"
+#include "zip/teleporter.h"
+#include "api/api.h"
+// ERRBUF_SIZE
+#include "config/dnsmasq_config.h"
+// inflate_buffer()
+#include "zip/gzip.h"
+// find_file_in_tar()
+#include "zip/tar.h"
+// sqlite3_open_v2()
+#include "database/sqlite3.h"
+// dbquery()
+#include "database/common.h"
+// MAX_ROTATIONS
+#include "files.h"
+
+#define MAXFILESIZE (50u*1024*1024)
+
+static int api_teleporter_GET(struct ftl_conn *api)
+{
+	mz_zip_archive zip = { 0 };
+	void *ptr = NULL;
+	size_t size = 0u;
+	char filename[128] = "";
+	const char *error = generate_teleporter_zip(&zip, filename, &ptr, &size);
+	if(error != NULL)
+		return send_json_error(api, 500,
+		                       "compression_error",
+		                       error,
+		                       NULL);
+
+	// Add header indicating that this is a file to be downloaded and stored as
+	// teleporter.zip (rather than showing the binary data in the browser
+	// window). This client is free to ignore and do whatever it wants with this
+	// data stream.
+	snprintf(pi_hole_extra_headers, sizeof(pi_hole_extra_headers),
+	         "Content-Disposition: attachment; filename=\"%s\"",
+	         filename);
+
+	// Send 200 OK with appropriate headers
+	mg_send_http_ok(api->conn, "application/zip", size);
+
+	// Clear extra headers
+	pi_hole_extra_headers[0] = '\0';
+
+	// Send raw (binary) ZIP content
+	mg_write(api->conn, ptr, size);
+
+	// Free allocated ZIP memory
+	free_teleporter_zip(&zip);
+
+	return 200;
+}
+
+// Struct to store the data we want to process
+struct upload_data {
+	bool too_large;
+	char *sid;
+	cJSON *import;
+	uint8_t *data;
+	char *filename;
+	size_t filesize;
+	struct {
+		bool file;
+		bool sid;
+		bool import;
+	} field;
+};
+
+// Callback function for CivetWeb to determine which fields we want to receive
+static int field_found(const char *key,
+                       const char *filename,
+                       char *path,
+                       size_t pathlen,
+                       void *user_data)
+{
+	struct upload_data *data = (struct upload_data *)user_data;
+	log_debug(DEBUG_API, "Found field: \"%s\", filename: \"%s\"", key, filename);
+
+	// Set all fields to false
+	memset(&data->field, false, sizeof(data->field));
+	if(strcasecmp(key, "file") == 0 && filename && *filename)
+	{
+		data->filename = strdup(filename);
+		data->field.file = true;
+		return MG_FORM_FIELD_STORAGE_GET;
+	}
+	else if(strcasecmp(key, "sid") == 0)
+	{
+		data->field.sid = true;
+		return MG_FORM_FIELD_STORAGE_GET;
+	}
+	else if(strcasecmp(key, "import") == 0)
+	{
+		data->field.import = true;
+		return MG_FORM_FIELD_STORAGE_GET;
+	}
+
+	// Ignore any other fields
+	return MG_FORM_FIELD_STORAGE_SKIP;
+}
+
+// Callback function for CivetWeb to receive the data of the fields we want to process.
+// This function might be called several times for the same field (large (> 8KB)
+// or chunked data), so we may need to append new data to existing data.
+static int field_get(const char *key, const char *value, size_t valuelen, void *user_data)
+{
+	struct upload_data *data = (struct upload_data *)user_data;
+	log_debug(DEBUG_API, "Received field: \"%s\" (length %zu bytes)", key, valuelen);
+
+	if(data->field.file)
+	{
+		if(data->filesize + valuelen > MAXFILESIZE)
+		{
+			log_warn("Uploaded Teleporter file is too large (limit is %u bytes)",
+			         MAXFILESIZE);
+			data->too_large = true;
+			return MG_FORM_FIELD_HANDLE_ABORT;
+		}
+		// Allocate memory for the raw file data
+		data->data = realloc(data->data, data->filesize + valuelen);
+		// Copy the raw file data
+		memcpy(data->data + data->filesize, value, valuelen);
+		// Store the size of the file raw data
+		data->filesize += valuelen;
+		log_debug(DEBUG_API, "Received file (%zu bytes, buffer is now %zu bytes)",
+		          valuelen, data->filesize);
+	}
+	else if(data->field.sid)
+	{
+		// Allocate memory for the SID
+		data->sid = calloc(valuelen + 1, sizeof(char));
+		// Copy the SID string
+		memcpy(data->sid, value, valuelen);
+		// Add terminating NULL byte (memcpy does not do this)
+		data->sid[valuelen] = '\0';
+	}
+	else if(data->field.import)
+	{
+		// Try to parse the JSON data
+		const char *json_error = NULL;
+		cJSON *json = cJSON_ParseWithLengthOpts(value, valuelen, &json_error, false);
+		if(json == NULL)
+		{
+			log_err("Unable to parse JSON data in API request, error at: %.20s", json_error);
+			return MG_FORM_FIELD_HANDLE_ABORT;
+		}
+
+		// Check if the JSON data is an object
+		if(!cJSON_IsObject(json))
+		{
+			log_err("JSON data in API request is not an object");
+			cJSON_Delete(json);
+			return MG_FORM_FIELD_HANDLE_ABORT;
+		}
+
+		// Store the parsed JSON data
+		data->import = json;
+	}
+
+	// If there is more data in this field, get the next chunk.
+	// Otherwise: handle the next field.
+	return MG_FORM_FIELD_HANDLE_GET;
+}
+
+// We don't use this function, but it is required by the CivetWeb API
+static int field_stored(const char *path, long long file_size, void *user_data)
+{
+	return 0;
+}
+
+static int free_upload_data(struct upload_data *data)
+{
+	// Free allocated memory
+	if(data->filename)
+	{
+		free(data->filename);
+		data->filename = NULL;
+	}
+	if(data->sid)
+	{
+		free(data->sid);
+		data->sid = NULL;
+	}
+	if(data->data)
+	{
+		free(data->data);
+		data->data = NULL;
+	}
+	if(data->import)
+	{
+		cJSON_Delete(data->import);
+		data->import = NULL;
+	}
+	return 0;
+}
+
+// Private function prototypes
+static int process_received_zip(struct ftl_conn *api, struct upload_data *data);
+static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *data);
+
+static int api_teleporter_POST(struct ftl_conn *api)
+{
+	struct upload_data data;
+	memset(&data, 0, sizeof(struct upload_data));
+	const struct mg_request_info *req_info = mg_get_request_info(api->conn);
+	struct mg_form_data_handler fdh = {field_found, field_get, field_stored, &data};
+
+	// Disallow large ZIP archives (> 50 MB) to prevent DoS attacks.
+	// Typically, the ZIP archive size should be around 30-100 kB.
+	if(req_info->content_length > MAXFILESIZE)
+	{
+		free_upload_data(&data);
+		return send_json_error(api, 400,
+		                       "bad_request",
+		                       "ZIP archive too large",
+		                       NULL);
+	}
+
+	// Call the form handler to process the POST request content
+	const int ret = mg_handle_form_request(api->conn, &fdh);
+	if(ret < 0)
+	{
+		free_upload_data(&data);
+		return send_json_error(api, 400,
+		                       "bad_request",
+		                       "Invalid form request",
+		                       NULL);
+	}
+
+	// Check if we received something we consider being a file
+	if(data.data == NULL || data.filesize == 0)
+	{
+		free_upload_data(&data);
+		return send_json_error(api, 400,
+		                       "bad_request",
+		                       "No ZIP archive received",
+		                       NULL);
+	}
+
+	// Check if the file we received is too large
+	if(data.too_large)
+	{
+		free_upload_data(&data);
+		return send_json_error(api, 400,
+		                       "bad_request",
+		                       "ZIP archive too large",
+		                       NULL);
+	}
+
+	// Check if we received something that claims to be a ZIP archive
+	// - filename should end in ".zip"
+	// - the data itself
+	//   - should be at least 40 bytes long
+	//   - start with 0x04034b50 (local file header signature, see https://pkware.cachefly.net/webdocs/APPNOTE/APPNOTE-6.3.9.TXT)
+	if(strlen(data.filename) > 4 &&
+	   strcmp(data.filename + strlen(data.filename) - 4, ".zip") == 0 &&
+	   data.filesize >= 40 &&
+	   memcmp(data.data, "\x50\x4b\x03\x04", 4) == 0)
+	{
+		return process_received_zip(api, &data);
+	}
+	// Check if we received something that claims to be a TAR.GZ archive
+	// - filename should end in ".tar.gz"
+	// - the data itself
+	//   - should be at least 40 bytes long
+	//   - start with 0x8b1f (local file header signature, see https://www.ietf.org/rfc/rfc1952.txt)
+	else if(strlen(data.filename) > 7 &&
+	        strcmp(data.filename + strlen(data.filename) - 7, ".tar.gz") == 0 &&
+	        data.filesize >= 40 &&
+	        memcmp(data.data, "\x1f\x8b", 2) == 0)
+	{
+		return process_received_tar_gz(api, &data);
+	}
+
+	// else: invalid file
+	free_upload_data(&data);
+	return send_json_error(api, 400,
+	                       "bad_request",
+	                       "Invalid file",
+	                       "The uploaded file does not appear to be a valid Pi-hole Teleporter archive");
+}
+
+static int process_received_zip(struct ftl_conn *api, struct upload_data *data)
+{
+	char hint[ERRBUF_SIZE];
+	memset(hint, 0, sizeof(hint));
+	cJSON *json_files = JSON_NEW_ARRAY();
+	const char *error = read_teleporter_zip(data->data, data->filesize, hint, data->import, json_files);
+	if(error != NULL)
+	{
+		const size_t msglen = strlen(error) + strlen(hint) + 4;
+		char *msg = calloc(msglen, sizeof(char));
+		strncpy(msg, error, msglen);
+		if(strlen(hint) > 0)
+		{
+			// Concatenate error message and hint into a single string
+			strcat(msg, ": ");
+			strcat(msg, hint);
+		}
+		free_upload_data(data);
+		return send_json_error_free(api, 400,
+		                            "bad_request",
+		                            "Invalid request",
+		                            msg, true);
+	}
+
+	// Free allocated memory
+	free_upload_data(data);
+
+	// Send response
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_ADD_ITEM_TO_OBJECT(json, "files", json_files);
+	JSON_SEND_OBJECT(json);
+}
+
+static struct teleporter_files {
+	const char *filename; // Filename of the file in the archive
+	const char *table_name; // Name of the table in the database
+	const int listtype; // Type of list (only used for domainlist table)
+	const size_t num_columns; // Number of columns in the table
+	const char *columns[10]; // List of columns in the table
+} teleporter_v5_files[] = {
+	{
+		.filename = "adlist.json",
+		.table_name = "adlist",
+		.listtype = -1,
+		.num_columns = 10,
+		.columns = { "id", "address", "enabled", "date_added", "date_modified", "comment", "date_updated", "number", "invalid_domains", "status" } // abp_entries and type are not defined in Pi-hole v5.x
+	},{
+		.filename = "adlist_by_group.json",
+		.table_name = "adlist_by_group",
+		.listtype = -1,
+		.num_columns = 2,
+		.columns = { "group_id", "adlist_id" }
+	},{
+		.filename = "blacklist.exact.json",
+		.table_name = "domainlist",
+		.listtype = 1, // GRAVITY_DOMAINLIST_DENY_EXACT
+		.num_columns = 7,
+		.columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" }
+	},{
+		.filename = "blacklist.regex.json",
+		.table_name = "domainlist",
+		.listtype = 3, // GRAVITY_DOMAINLIST_DENY_REGEX
+		.num_columns = 7,
+		.columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" }
+	},{
+		.filename = "client.json",
+		.table_name = "client",
+		.listtype = -1,
+		.num_columns = 5,
+		.columns = { "id", "ip", "date_added", "date_modified", "comment" }
+	},{
+		.filename = "client_by_group.json",
+		.table_name = "client_by_group",
+		.listtype = -1,
+		.num_columns = 2,
+		.columns = { "group_id", "client_id" }
+	},{
+		.filename = "domainlist_by_group.json",
+		.table_name = "domainlist_by_group",
+		.listtype = -1,
+		.num_columns = 2,
+		.columns = { "group_id", "domainlist_id" }
+	},{
+		.filename = "group.json",
+		.table_name = "group",
+		.listtype = -1,
+		.num_columns = 6,
+		.columns = { "id", "enabled", "name", "date_added", "date_modified", "description" }
+	},{
+		.filename = "whitelist.exact.json",
+		.table_name = "domainlist",
+		.listtype = 0, // GRAVITY_DOMAINLIST_ALLOW_EXACT
+		.num_columns = 7,
+		.columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" }
+	},{
+		.filename = "whitelist.regex.json",
+		.table_name = "domainlist",
+		.listtype = 2, // GRAVITY_DOMAINLIST_ALLOW_REGEX
+		.num_columns = 7,
+		.columns = { "id", "domain", "enabled", "date_added", "date_modified", "comment", "type" }
+	}
+};
+
+static bool import_json_table(cJSON *json, struct teleporter_files *file)
+{
+	// Check if the JSON object is an array
+	if(!cJSON_IsArray(json))
+	{
+		log_err("import_json_table(%s): JSON object is not an array", file->filename);
+		return false;
+	}
+
+	// Check if the JSON array is empty, if so, we can return early
+	const int num_entries = cJSON_GetArraySize(json);
+
+	// Check if all the JSON entries contain all the expected columns
+	cJSON *json_object = NULL;
+	cJSON_ArrayForEach(json_object, json)
+	{
+		if(!cJSON_IsObject(json_object))
+		{
+			log_err("import_json_table(%s): JSON array does not contain objects", file->filename);
+			return false;
+		}
+
+		// If this is a record for the domainlist table, add type/kind
+		if(strcmp(file->table_name, "domainlist") == 0)
+		{
+			// Add type/kind to the JSON object
+			cJSON_AddNumberToObject(json_object, "type", file->listtype);
+		}
+
+		// Check if the JSON object contains the expected columns
+		for(size_t i = 0; i < file->num_columns; i++)
+		{
+			if(cJSON_GetObjectItemCaseSensitive(json_object, file->columns[i]) == NULL)
+			{
+				log_err("import_json_table(%s): JSON object does not contain column \"%s\"", file->filename, file->columns[i]);
+				return false;
+			}
+		}
+	}
+
+	log_info("import_json_table(%s): JSON array contains %d entr%s", file->filename, num_entries, num_entries == 1 ? "y" : "ies");
+
+	// Open database connection
+	sqlite3 *db = NULL;
+	if(sqlite3_open_v2(config.files.gravity.v.s, &db, SQLITE_OPEN_READWRITE, NULL) != SQLITE_OK)
+	{
+		log_err("import_json_table(%s): Unable to open database file \"%s\": %s",
+		        file->filename, config.files.database.v.s, sqlite3_errmsg(db));
+		sqlite3_close(db);
+		return false;
+	}
+
+	// Disable foreign key constraints
+	if(sqlite3_exec(db, "PRAGMA foreign_keys = OFF;", NULL, NULL, NULL) != SQLITE_OK)
+	{
+		log_err("import_json_table(%s): Unable to disable foreign key constraints: %s", file->filename, sqlite3_errmsg(db));
+		sqlite3_close(db);
+		return false;
+	}
+
+	// Start transaction
+	if(sqlite3_exec(db, "BEGIN TRANSACTION;", NULL, NULL, NULL) != SQLITE_OK)
+	{
+		log_err("import_json_table(%s): Unable to start transaction: %s", file->filename, sqlite3_errmsg(db));
+		sqlite3_close(db);
+		return false;
+	}
+
+	// Clear existing table entries
+	if(file->listtype < 0)
+	{
+		// Delete all entries in the table
+		log_debug(DEBUG_API, "import_json_table(%s): Deleting all entries from table \"%s\"", file->filename, file->table_name);
+		if(dbquery(db, "DELETE FROM \"%s\";", file->table_name) != SQLITE_OK)
+		{
+			log_err("import_json_table(%s): Unable to delete entries from table \"%s\": %s",
+			        file->filename, file->table_name, sqlite3_errmsg(db));
+			sqlite3_close(db);
+			return false;
+		}
+	}
+	else
+	{
+		// Delete all entries in the table of the same type
+		log_debug(DEBUG_API, "import_json_table(%s): Deleting all entries from table \"%s\" of type %d", file->filename, file->table_name, file->listtype);
+		if(dbquery(db, "DELETE FROM \"%s\" WHERE type = %d;", file->table_name, file->listtype) != SQLITE_OK)
+		{
+			log_err("import_json_table(%s): Unable to delete entries from table \"%s\": %s",
+			        file->filename, file->table_name, sqlite3_errmsg(db));
+			sqlite3_close(db);
+			return false;
+		}
+	}
+
+	// Build dynamic SQL insertion statement
+	// "INSERT OR IGNORE INTO table (column1, column2, ...) VALUES (?, ?, ...);"
+	char *sql = sqlite3_mprintf("INSERT OR IGNORE INTO \"%s\" (", file->table_name);
+	for(size_t i = 0; i < file->num_columns; i++)
+	{
+		char *sql2 = sqlite3_mprintf("%s%s", sql, file->columns[i]);
+		sqlite3_free(sql);
+		sql = NULL;
+		if(i < file->num_columns - 1)
+		{
+			sql = sqlite3_mprintf("%s, ", sql2);
+			sqlite3_free(sql2);
+			sql2 = NULL;
+		}
+		else
+		{
+			sql = sqlite3_mprintf("%s) VALUES (", sql2);
+			sqlite3_free(sql2);
+			sql2 = NULL;
+		}
+	}
+	for(size_t i = 0; i < file->num_columns; i++)
+	{
+		char *sql2 = sqlite3_mprintf("%s?", sql);
+		sqlite3_free(sql);
+		sql = NULL;
+		if(i < file->num_columns - 1)
+		{
+			sql = sqlite3_mprintf("%s, ", sql2);
+			sqlite3_free(sql2);
+			sql2 = NULL;
+		}
+		else
+		{
+			sql = sqlite3_mprintf("%s);", sql2);
+			sqlite3_free(sql2);
+			sql2 = NULL;
+		}
+	}
+
+	// Prepare SQL statement
+	sqlite3_stmt *stmt = NULL;
+	if(sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK)
+	{
+		log_err("Unable to prepare SQL statement: %s", sqlite3_errmsg(db));
+		sqlite3_free(sql);
+		sqlite3_close(db);
+		return false;
+	}
+
+	// Free allocated memory
+	sqlite3_free(sql);
+	sql = NULL;
+
+	// Iterate over all JSON objects
+	cJSON_ArrayForEach(json_object, json)
+	{
+		// Bind values to SQL statement
+		for(size_t i = 0; i < file->num_columns; i++)
+		{
+			cJSON *json_value = cJSON_GetObjectItemCaseSensitive(json_object, file->columns[i]);
+			if(cJSON_IsString(json_value))
+			{
+				// Bind string value
+				if(sqlite3_bind_text(stmt, i + 1, json_value->valuestring, -1, SQLITE_STATIC) != SQLITE_OK)
+				{
+					log_err("Unable to bind text value to SQL statement: %s", sqlite3_errmsg(db));
+					sqlite3_finalize(stmt);
+					sqlite3_close(db);
+					return false;
+				}
+			}
+			else if(cJSON_IsNumber(json_value))
+			{
+				// Bind integer value
+				if(sqlite3_bind_int(stmt, i + 1, json_value->valueint) != SQLITE_OK)
+				{
+					log_err("Unable to bind integer value to SQL statement: %s", sqlite3_errmsg(db));
+					sqlite3_finalize(stmt);
+					sqlite3_close(db);
+					return false;
+				}
+			}
+			else if(cJSON_IsNull(json_value))
+			{
+				// Bind NULL value
+				if(sqlite3_bind_null(stmt, i + 1) != SQLITE_OK)
+				{
+					log_err("Unable to bind NULL value to SQL statement: %s", sqlite3_errmsg(db));
+					sqlite3_finalize(stmt);
+					sqlite3_close(db);
+					return false;
+				}
+			}
+			else
+			{
+				log_err("Unable to bind value to SQL statement: type = %X", (unsigned int)json_value->type & 0xFF);
+				sqlite3_finalize(stmt);
+				sqlite3_close(db);
+				return false;
+			}
+		}
+
+		// Execute SQL statement
+		if(sqlite3_step(stmt) != SQLITE_DONE)
+		{
+			log_err("Unable to execute SQL statement: %s", sqlite3_errmsg(db));
+			sqlite3_finalize(stmt);
+			sqlite3_close(db);
+			return false;
+		}
+
+		// Reset SQL statement
+		if(sqlite3_reset(stmt) != SQLITE_OK)
+		{
+			log_err("Unable to reset SQL statement: %s", sqlite3_errmsg(db));
+			sqlite3_finalize(stmt);
+			sqlite3_close(db);
+			return false;
+		}
+	}
+
+	// Finalize SQL statement
+	if(sqlite3_finalize(stmt) != SQLITE_OK)
+	{
+		log_err("Unable to finalize SQL statement: %s", sqlite3_errmsg(db));
+		sqlite3_close(db);
+		return false;
+	}
+
+	// Commit transaction
+	if(sqlite3_exec(db, "COMMIT;", NULL, NULL, NULL) != SQLITE_OK)
+	{
+		log_err("Unable to commit transaction: %s", sqlite3_errmsg(db));
+		sqlite3_close(db);
+		return false;
+	}
+
+	// Close database connection
+	sqlite3_close(db);
+
+	return true;
+}
+
+static int process_received_tar_gz(struct ftl_conn *api, struct upload_data *data)
+{
+	// Try to decompress the received data
+	uint8_t *archive = NULL;
+	mz_ulong archive_size = 0u;
+	if(!inflate_buffer(data->data, data->filesize, &archive, &archive_size))
+	{
+		free_upload_data(data);
+		return send_json_error(api, 400,
+		                       "bad_request",
+		                       "Invalid GZIP archive",
+		                       "The uploaded file does not appear to be a valid gzip archive - decompression failed");
+	}
+
+	// Print all files in the TAR archive if in debug mode
+	if(config.debug.api.v.b)
+	{
+		cJSON *json_files = list_files_in_tar(archive, archive_size);
+
+		cJSON *file = NULL;
+		cJSON_ArrayForEach(file, json_files)
+		{
+			const cJSON *name = cJSON_GetObjectItemCaseSensitive(file, "name");
+			const cJSON *size = cJSON_GetObjectItemCaseSensitive(file, "size");
+			if(name == NULL || size == NULL)
+				continue;
+
+			log_debug(DEBUG_API, "Found file in TAR archive: \"%s\" (%d bytes)",
+			          name->valuestring, size->valueint);
+		}
+	}
+
+	// Parse JSON files in the TAR archive
+	cJSON *imported_files = JSON_NEW_ARRAY();
+
+	// Check if the archive contains gravity tables
+	cJSON *gravity = data->import != NULL ? cJSON_GetObjectItemCaseSensitive(data->import, "gravity") : NULL;
+	for(size_t i = 0; i < sizeof(teleporter_v5_files) / sizeof(struct teleporter_files); i++)
+	{
+		// - if import is non-NULL we may skip some tables
+		if(data->import != NULL)
+		{
+			// - if import is non-NULL, but gravity is NULL we skip
+			//   the import of gravity tables altogether
+			// - if import is non-NULL, and gravity is non-NULL, we
+			//   import the file/table if it is in the object, a
+			//   boolean and true
+			if(gravity == NULL || !JSON_KEY_TRUE(gravity, teleporter_v5_files[i].table_name))
+			{
+				log_info("Skipping import of \"%s\" as it was not requested for import (JSON: %s, gravity: %s)",
+				         teleporter_v5_files[i].filename,
+				         data->import != NULL ? "yes" : "no",
+				         gravity != NULL ? "yes" : "no");
+				continue;
+			}
+		}
+
+		// Import the JSON file
+		size_t fileSize = 0u;
+		cJSON *json = NULL;
+		const char *file = find_file_in_tar(archive, archive_size, teleporter_v5_files[i].filename, &fileSize);
+		const char *json_error = NULL;
+		if(file != NULL && fileSize > 0u && (json = cJSON_ParseWithLengthOpts(file, fileSize, &json_error, false)) != NULL)
+		{
+			if(import_json_table(json, &teleporter_v5_files[i]))
+				JSON_COPY_STR_TO_ARRAY(imported_files, teleporter_v5_files[i].filename);
+		}
+		else if(json_error != NULL)
+		{
+			log_err("Unable to parse JSON file \"%s\", error at: %.20s",
+			        teleporter_v5_files[i].filename, json_error);
+		}
+		else
+		{
+			log_debug(DEBUG_CONFIG, "Unable to find file \"%s\" in TAR archive",
+			          teleporter_v5_files[i].filename);
+		}
+	}
+
+	// Temporarily write further files to to disk so we can import them on restart
+	struct {
+		const char *archive_name;
+		const char *destination;
+	} extract_files[] = {
+		{
+			// i = 0
+			.archive_name = "custom.list",
+			.destination = DNSMASQ_CUSTOM_LIST_LEGACY
+		},{
+			// i = 1
+			.archive_name = "dhcp.leases",
+			.destination = DHCPLEASESFILE
+		},{
+			// i = 2
+			.archive_name = "pihole-FTL.conf",
+			.destination = GLOBALCONFFILE_LEGACY
+		},{
+			// i = 3
+			.archive_name = "setupVars.conf",
+			.destination = config.files.setupVars.v.s
+		}
+	};
+	for(size_t i = 0; i < sizeof(extract_files) / sizeof(*extract_files); i++)
+	{
+		size_t fileSize = 0u;
+		const char *file = find_file_in_tar(archive, archive_size, extract_files[i].archive_name, &fileSize);
+
+		if(data->import != NULL && i == 1 && !JSON_KEY_TRUE(data->import, "dhcp_leases"))
+		{
+			log_info("Skipping import of \"%s\" as it was not requested for import",
+			         extract_files[i].archive_name);
+			continue;
+		}
+		// all other values of i belong to config files
+		else if(data->import != NULL && !JSON_KEY_TRUE(data->import, "config"))
+		{
+			log_info("Skipping import of \"%s\" as it was not requested for import",
+			         extract_files[i].archive_name);
+			continue;
+		}
+
+		if(file != NULL && fileSize > 0u)
+		{
+			// Write file to disk
+			log_info("Writing file \"%s\" (%zu bytes) to \"%s\"",
+			         extract_files[i].archive_name, fileSize, extract_files[i].destination);
+			FILE *fp = fopen(extract_files[i].destination, "wb");
+			if(fp == NULL)
+			{
+				log_err("Unable to open file \"%s\" for writing: %s", extract_files[i].destination, strerror(errno));
+				continue;
+			}
+			if(fwrite(file, fileSize, 1, fp) != 1)
+			{
+				log_err("Unable to write file \"%s\": %s", extract_files[i].destination, strerror(errno));
+				fclose(fp);
+				continue;
+			}
+			fclose(fp);
+			JSON_COPY_STR_TO_ARRAY(imported_files, extract_files[i].destination);
+		}
+	}
+
+	// Append WEB_PORTS to setupVars.conf
+	FILE *fp = fopen(config.files.setupVars.v.s, "a");
+	if(fp == NULL)
+		log_err("Unable to open file \"%s\" for appending: %s", config.files.setupVars.v.s, strerror(errno));
+	else
+	{
+		fprintf(fp, "WEB_PORTS=%s\n", config.webserver.port.v.s);
+		fclose(fp);
+	}
+
+	// Remove pihole.toml to prevent it from being imported on restart
+	if(remove(GLOBALTOMLPATH) != 0)
+		log_err("Unable to remove file \"%s\": %s", GLOBALTOMLPATH, strerror(errno));
+
+	// Remove all rotated pihole.toml files to avoid automatic config
+	// restore on restart
+	for(unsigned int i = MAX_ROTATIONS; i > 0; i--)
+	{
+		const char *fname = GLOBALTOMLPATH;
+		const char *filename = basename(fname);
+		// extra 6 bytes is enough space for up to 999 rotations ("/", ".", "\0", "999")
+		const size_t buflen = strlen(filename) + strlen(BACKUP_DIR) + 6;
+		char *path = calloc(buflen, sizeof(char));
+		snprintf(path, buflen, BACKUP_DIR"/%s.%u", filename, i);
+
+		// Remove file (if it exists)
+		if(remove(path) != 0 && errno != ENOENT)
+			log_err("Unable to remove file \"%s\": %s", path, strerror(errno));
+	}
+
+	// Free allocated memory
+	free_upload_data(data);
+
+	// Signal FTL we want to restart for re-import
+	api->ftl.restart = true;
+
+	// Send response
+	cJSON *json = JSON_NEW_OBJECT();
+	JSON_ADD_ITEM_TO_OBJECT(json, "files", imported_files);
+	JSON_SEND_OBJECT(json);
+}
+
+int api_teleporter(struct ftl_conn *api)
+{
+	if(api->method == HTTP_GET)
+		return api_teleporter_GET(api);
+	if(api->method == HTTP_POST)
+		return api_teleporter_POST(api);
+
+	return 0;
+}

src/api/theme.c

@@ -0,0 +1,89 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2023 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  Theme-related routines
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+// NULL
+#include <stddef.h>
+// strcasecmp()
+#include <string.h>
+
+#include "theme.h"
+
+struct web_themes webthemes[THEME_MAX] = {
+	{
+		/* id */ THEME_DEFAULT_AUTO,
+		/* name */ "default-auto",
+		/* description */ "Pi-hole auto",
+		/* dark */ true,
+		/* color */ "#367fa9"
+	},
+	{
+		/* id */ THEME_DEFAULT_LIGHT,
+		/* name */ "default-light",
+		/* description */ "Pi-hole day",
+		/* dark */ false,
+		/* color */ "#367fa9"
+	},
+	{
+		/* id */ THEME_DEFAULT_DARK,
+		/* name */ "default-dark",
+		/* description */ "Pi-hole midnight",
+		/* dark */ true,
+		/* color */ "#272c30"
+	},
+	{
+		/* id */ THEME_DEFAULT_DARKER,
+		/* name */ "default-darker",
+		/* description */ "Pi-hole deep-midnight",
+		/* dark */ true,
+		/* color */ "#2e6786"
+	},
+	{
+		/* id */ THEME_HIGH_CONTRAST,
+		/* name */ "high-contrast",
+		/* description */ "High-contrast light",
+		/* dark */ false,
+		/* color */ "#0078a0"
+	},
+	{
+		/* id */ THEME_HIGH_CONTRAST_DARK,
+		/* name */ "high-contrast-dark",
+		/* description */ "High-contrast dark",
+		/* dark */ true,
+		/* color */ "#0077c7"
+	},
+	{
+		/* id */ THEME_LCARS,
+		/* name */ "lcars",
+		/* description */ "Star Trek LCARS",
+		/* dark */ true,
+		/* color */ "#4488FF"
+	},
+};
+
+const char * __attribute__ ((pure)) get_web_theme_str(const enum web_theme web_theme)
+{
+	for(enum web_theme i = 0; i < THEME_MAX; i++)
+		if(webthemes[i].id == web_theme)
+			return webthemes[i].name;
+	return NULL;
+}
+
+int __attribute__ ((pure)) get_web_theme_val(const char *web_theme)
+{
+	// Iterate over all possible theme values
+	for(enum web_theme i = 0; i < THEME_MAX; i++)
+	{
+		if(strcasecmp(web_theme, webthemes[i].name) == 0)
+			return i;
+	}
+
+	// Invalid value
+	return -1;
+}

src/api/theme.h

@@ -0,0 +1,41 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2017 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  API route prototypes
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+#ifndef THEME_H
+#define THEME_H
+
+#include <stdbool.h>
+
+enum web_theme {
+	THEME_DEFAULT_AUTO = 0,
+	THEME_DEFAULT_LIGHT,
+	THEME_DEFAULT_DARK,
+	THEME_DEFAULT_DARKER,
+	THEME_HIGH_CONTRAST,
+	THEME_HIGH_CONTRAST_DARK,
+	THEME_LCARS,
+	THEME_MAX // This needs to be the last element in this enum
+} __attribute__ ((packed));
+
+struct web_themes{
+	const enum web_theme id;
+	const char *name;
+	const char *description;
+	const bool dark;
+	const char *color;
+};
+
+// defined in theme.c
+extern struct web_themes webthemes[THEME_MAX];
+
+// Prototypes
+const char * __attribute__ ((pure)) get_web_theme_str(const enum web_theme web_theme);
+int __attribute__ ((pure)) get_web_theme_val(const char *web_theme);
+
+#endif // THEME_H

src/args.c

@@ -19,6 +19,10 @@
 #  define NETTLE_VERSION_MINOR 0
 #endif
 
+#ifdef HAVE_MBEDTLS
+#include <mbedtls/version.h>
+#endif
+
 #include "FTL.h"
 #include "args.h"
 #include "version.h"
@@ -36,8 +40,33 @@
 #include "tools/gravity-parseList.h"
 // run_dhcp_discover()
 #include "tools/dhcp-discover.h"
+// mg_version()
+#include "webserver/civetweb/civetweb.h"
+// cJSON_Version()
+#include "webserver/cJSON/cJSON.h"
+#include "config/cli.h"
+#include "config/config.h"
+// compression functions
+#include "zip/gzip.h"
+// teleporter functions
+#include "zip/teleporter.h"
+// printTOTP()
+#include "api/api.h"
+// generate_certificate()
+#include "webserver/x509.h"
+// run_dhcp_discover()
+#include "tools/dhcp-discover.h"
 // run_arp_scan()
 #include "tools/arp-scan.h"
+// run_performance_test()
+#include "config/password.h"
+// idn2_to_ascii_lz()
+#include <idn2.h>
+// sha256sum()
+#include "files.h"
+// resolveHostname()
+#include "resolve.h"
+
 // defined in dnsmasq.c
 extern void print_dnsmasq_version(const char *yellow, const char *green, const char *bold, const char *normal);
 
@@ -74,7 +103,7 @@ const char** argv_dnsmasq = NULL;
 #define COL_PURPLE	"\x1b[95m" // bright foreground color
 #define COL_CYAN	"\x1b[96m" // bright foreground color
 
-static inline bool __attribute__ ((pure)) is_term(void)
+static bool __attribute__ ((pure)) is_term(void)
 {
 	// test whether STDOUT refers to a terminal
 	return isatty(fileno(stdout)) == 1;
@@ -175,15 +204,203 @@ void parse_args(int argc, char* argv[])
 	   (argc > 1 && strEndsWith(argv[1], ".db")))
 			exit(sqlite3_shell_main(argc, argv));
 
+	// Compression feature
+	if((argc == 3 || argc == 4) &&
+	   (strcmp(argv[1], "gzip") == 0 || strcmp(argv[1], "--gzip") == 0))
+	{
+		// Enable stdout printing
+		cli_mode = true;
+		log_ctrl(false, true);
+
+		// Get input and output file names
+		const char *infile = argv[2];
+		bool is_gz = strEndsWith(infile, ".gz");
+		char *outfile = NULL;
+		if(argc == 4)
+		{
+			// If an output file is given, we use it
+			outfile = strdup(argv[3]);
+		}
+		else if(is_gz)
+		{
+			// If no output file is given, and this is a gzipped
+			// file, we use the input file name without ".gz"
+			// appended
+			outfile = calloc(strlen(infile)-2, sizeof(char));
+			memcpy(outfile, infile, strlen(infile)-3);
+		}
+		else
+		{
+			// If no output file is given, and this is not a gzipped
+			// file, we use the input file name with ".gz" appended
+			outfile = calloc(strlen(infile)+4, sizeof(char));
+			strcpy(outfile, infile);
+			strcat(outfile, ".gz");
+		}
+
+		bool success = false;
+		if(is_gz)
+		{
+			// If the input file is already gzipped, we decompress it
+			success = inflate_file(infile, outfile, true);
+		}
+		else
+		{
+			// If the input file is not gzipped, we compress it
+			success = deflate_file(infile, outfile, true);
+		}
+
+		// Free allocated memory
+		free(outfile);
+
+		// Return exit code
+		exit(success ? EXIT_SUCCESS : EXIT_FAILURE);
+	}
+
+	// Set config option through CLI
+	if(argc > 1 && strcmp(argv[1], "--config") == 0)
+	{
+		// Enable stdout printing
+		cli_mode = true;
+		log_ctrl(false, false);
+		readFTLconf(&config, false);
+		log_ctrl(false, true);
+		clear_debug_flags(); // No debug printing wanted
+		if(argc == 2)
+			exit(get_config_from_CLI(NULL, false));
+		else if(argc == 3)
+			exit(get_config_from_CLI(argv[2], false));
+		else if(argc == 4 && strcmp(argv[2], "-q") == 0)
+			exit(get_config_from_CLI(argv[3], true));
+		else if(argc == 4)
+			exit(set_config_from_CLI(argv[2], argv[3]));
+		else
+		{
+			printf("Usage: %s --config [<config item key>] [<value>]\n", argv[0]);
+			printf("Example: %s --config dns.blockESNI true\n", argv[0]);
+			exit(EXIT_FAILURE);
+		}
+	}
+
+
+	// Set config option through CLI
+	if(argc == 2 && strcmp(argv[1], "--totp") == 0)
+	{
+		cli_mode = true;
+		log_ctrl(false, false);
+		readFTLconf(&config, false);
+		log_ctrl(false, true);
+		clear_debug_flags(); // No debug printing wanted
+		exit(printTOTP());
+	}
+
+
+	// Create teleporter archive through CLI
+	if(argc == 2 && strcmp(argv[1], "--teleporter") == 0)
+	{
+		// Enable stdout printing
+		cli_mode = true;
+		log_ctrl(false, true);
+		readFTLconf(&config, false);
+		exit(write_teleporter_zip_to_disk() ? EXIT_SUCCESS : EXIT_FAILURE);
+	}
+
+	// Import teleporter archive through CLI
+	if(argc == 3 && strcmp(argv[1], "--teleporter") == 0)
+	{
+		// Enable stdout printing
+		cli_mode = true;
+		log_ctrl(false, true);
+		readFTLconf(&config, false);
+		exit(read_teleporter_zip_from_disk(argv[2]) ? EXIT_SUCCESS : EXIT_FAILURE);
+	}
+
+	// Generate X.509 certificate
+	if(argc > 1 && strcmp(argv[1], "--gen-x509") == 0)
+	{
+		if(argc < 3 || argc > 5)
+		{
+			printf("Usage: %s --gen-x509 <output file> [<domain>] [rsa]\n", argv[0]);
+			printf("Example:          %s --gen-x509 /etc/pihole/tls.pem\n", argv[0]);
+			printf(" with domain:     %s --gen-x509 /etc/pihole/tls.pem pi.hole\n", argv[0]);
+			printf(" RSA with domain: %s --gen-x509 /etc/pihole/tls.pem nanopi.lan rsa\n", argv[0]);
+			exit(EXIT_FAILURE);
+		}
+		// Enable stdout printing
+		cli_mode = true;
+		log_ctrl(false, true);
+		const char *domain = argc > 3 ? argv[3] : "pi.hole";
+		const bool rsa = argc > 4 && strcasecmp(argv[4], "rsa") == 0;
+		exit(generate_certificate(argv[2], rsa, domain) ? EXIT_SUCCESS : EXIT_FAILURE);
+	}
+
+	// Parse X.509 certificate
+	if(argc > 1 &&
+	  (strcmp(argv[1], "--read-x509") == 0 ||
+	   strcmp(argv[1], "--read-x509-key") == 0))
+	{
+		if(argc > 4)
+		{
+			printf("Usage: %s %s [<input file>] [<domain>]\n", argv[0], argv[1]);
+			printf("Example: %s %s /etc/pihole/tls.pem\n", argv[0], argv[1]);
+			printf(" with domain: %s %s /etc/pihole/tls.pem pi.hole\n", argv[0], argv[1]);
+			exit(EXIT_FAILURE);
+		}
+
+		// Option parsing
+		// Should we report on the private key?
+		const bool private_key = strcmp(argv[1], "--read-x509-key") == 0;
+		// If no certificate file is given, we use the one from the config
+		const char *certfile = NULL;
+		if(argc == 2)
+		{
+			readFTLconf(&config, false);
+			certfile = config.webserver.tls.cert.v.s;
+		}
+		else
+			certfile = argv[2];
+
+		// If no domain is given, we only check the certificate
+		const char *domain = argc > 3 ? argv[3] : NULL;
+
+		// Enable stdout printing
+		cli_mode = true;
+		log_ctrl(false, true);
+
+		enum cert_check result = read_certificate(certfile, domain, private_key);
+
+		if(argc < 4)
+			exit(result == CERT_OKAY ? EXIT_SUCCESS : EXIT_FAILURE);
+		else if(result == CERT_DOMAIN_MATCH)
+		{
+			printf("Certificate matches domain %s\n", argv[3]);
+			exit(EXIT_SUCCESS);
+		}
+		else
+		{
+			printf("Certificate does not match domain %s\n", argv[3]);
+			exit(EXIT_FAILURE);
+		}
+	}
+
 	// If the first argument is "gravity" (e.g., /usr/bin/pihole-FTL gravity),
 	// we offer some specialized gravity tools
-	if(argc > 1 && strcmp(argv[1], "gravity") == 0)
+	if(argc > 1 && (strcmp(argv[1], "gravity") == 0 || strcmp(argv[1], "antigravity") == 0))
 	{
+		const bool antigravity = strcmp(argv[1], "antigravity") == 0;
+
 		// pihole-FTL gravity parseList <infile> <outfile> <adlistID>
-		if(argc == 6 && strcmp(argv[2], "parseList") == 0)
+		if(argc == 6 && strcasecmp(argv[2], "parseList") == 0)
 		{
 			// Parse the given list and write the result to the given file
-			exit(gravity_parseList(argv[3], argv[4], argv[5]));
+			exit(gravity_parseList(argv[3], argv[4], argv[5], false, antigravity));
+		}
+
+		// pihole-FTL gravity checkList <infile>
+		if(argc == 4 && strcasecmp(argv[2], "checkList") == 0)
+		{
+			// Parse the given list and write the result to the given file
+			exit(gravity_parseList(argv[3], "", "-1", true, antigravity));
 		}
 
 		printf("Incorrect usage of pihole-FTL gravity subcommand\n");
@@ -198,6 +415,14 @@ void parse_args(int argc, char* argv[])
 		exit(run_dhcp_discover());
 	}
 
+	// Password hashing performance test
+	if(argc > 1 && (strcmp(argv[1], "--perf") == 0 || strcmp(argv[1], "performance") == 0))
+	{
+		// Enable stdout printing
+		cli_mode = true;
+		exit(run_performance_test());
+	}
+
 	// ARP scanning mode
 	if(argc > 1 && strcmp(argv[1], "arp-scan") == 0)
 	{
@@ -208,6 +433,90 @@ void parse_args(int argc, char* argv[])
 		exit(run_arp_scan(scan_all, extreme_mode));
 	}
 
+	// IDN2 conversion mode
+	if(argc > 1 && strcmp(argv[1], "idn2") == 0)
+	{
+		// Enable stdout printing
+		cli_mode = true;
+		if(argc == 3)
+		{
+			// Convert unicode domain to punycode
+			char *punycode = NULL;
+			const int rc = idn2_to_ascii_lz(argv[2], &punycode, IDN2_NFC_INPUT | IDN2_NONTRANSITIONAL);
+			if (rc != IDN2_OK)
+			{
+				// Invalid domain name
+				printf("Invalid domain name: %s\n", argv[2]);
+				exit(EXIT_FAILURE);
+			}
+
+			// Convert punycode domain to lowercase
+			for(unsigned int i = 0u; i < strlen(punycode); i++)
+				punycode[i] = tolower(punycode[i]);
+
+			printf("%s\n", punycode);
+			exit(EXIT_SUCCESS);
+
+		}
+		else if(argc == 4 && (strcmp(argv[2], "-d") == 0 || strcmp(argv[2], "--decode") == 0))
+		{
+			// Convert punycode domain to unicode
+			char *unicode = NULL;
+			const int rc = idn2_to_unicode_lzlz(argv[3], &unicode, IDN2_NFC_INPUT | IDN2_NONTRANSITIONAL);
+			if (rc != IDN2_OK)
+			{
+				// Invalid domain name
+				printf("Invalid domain name: %s\n", argv[3]);
+				exit(EXIT_FAILURE);
+			}
+
+			printf("%s\n", unicode);
+			exit(EXIT_SUCCESS);
+		}
+		else
+		{
+			printf("Usage: %s idn2 [--decode] <domain>\n", argv[0]);
+			exit(EXIT_FAILURE);
+		}
+	}
+
+	// sha256sum mode
+	if(argc == 3 && strcmp(argv[1], "sha256sum") == 0)
+	{
+		// Enable stdout printing
+		cli_mode = true;
+		uint8_t checksum[SHA256_DIGEST_SIZE];
+		if(!sha256sum(argv[2], checksum))
+			exit(EXIT_FAILURE);
+
+		// Convert checksum to hex string
+		char hex[SHA256_DIGEST_SIZE*2+1];
+		sha256_raw_to_hex(checksum, hex);
+
+		// Print result
+		printf("%s  %s\n", hex, argv[2]);
+		exit(EXIT_SUCCESS);
+	}
+
+	// Local reverse name resolver
+	if(argc == 3 && strcasecmp(argv[1], "ptr") == 0)
+	{
+		// Enable stdout printing
+		cli_mode = true;
+
+		// Need to get dns.port and the resolver settings
+		readFTLconf(&config, false);
+
+		char *name = resolveHostname(argv[2], true);
+		if(name == NULL)
+			exit(EXIT_FAILURE);
+
+		// Print result
+		printf("%s\n", name);
+		free(name);
+		exit(EXIT_SUCCESS);
+	}
+
 	// start from 1, as argv[0] is the executable name
 	for(int i = 1; i < argc; i++)
 	{
@@ -276,7 +585,23 @@ void parse_args(int argc, char* argv[])
 			const char *arg[2];
 			arg[0] = "";
 			arg[1] = "--test";
-			exit(main_dnsmasq(2, arg));
+			log_ctrl(false, true);
+			exit(main_dnsmasq(2, (char**)arg));
+		}
+
+		// Implement dnsmasq's test function, no need to prepare the entire FTL
+		// environment (initialize shared memory, lead queries from long-term
+		// database, ...) when the task is a simple (dnsmasq) syntax check
+		if(argc == 3 && strcmp(argv[1], "dnsmasq-test-file") == 0)
+		{
+			const char *arg[3];
+			char *filename = calloc(strlen(argv[2])+strlen("--conf-file=")+1, sizeof(char));
+			arg[0] = "";
+			sprintf(filename, "--conf-file=%s", argv[2]);
+			arg[1] = filename;
+			arg[2] = "--test";
+			log_ctrl(false, true);
+			exit(main_dnsmasq(3, (char**)arg));
 		}
 
 		// If we find "--" we collect everything behind that for dnsmasq
@@ -379,6 +704,7 @@ void parse_args(int argc, char* argv[])
 			const char *bold = cli_bold();
 			const char *normal = cli_normal();
 			const char *green = cli_color(COL_GREEN);
+			const char *red = cli_color(COL_RED);
 			const char *yellow = cli_color(COL_YELLOW);
 
 			// Print FTL version
@@ -421,6 +747,69 @@ void parse_args(int argc, char* argv[])
 			printf("Version:         %s%s" xstr(NETTLE_VERSION_MAJOR) "." xstr(NETTLE_VERSION_MINOR) "%s\n",
 			       green, bold, normal);
 			printf("GMP:             %s\n", NETTLE_USE_MINI_GMP ? "Mini" : "Full");
+			printf("\n");
+			printf("****************************** %s%sCivetWeb%s *****************************\n",
+			       yellow, bold, normal);
+#ifdef MBEDTLS_VERSION_STRING_FULL
+			printf("Version:         %s%s%s%s with %smbed TLS %s%s"MBEDTLS_VERSION_STRING"%s\n",
+			       green, bold, mg_version(), normal, yellow, green, bold, normal);
+#else
+			printf("Version:         %s%s%s%s\n", green, bold, mg_version(), normal);
+#endif
+			printf("Features:        ");
+			if(mg_check_feature(MG_FEATURES_FILES))
+				printf("Files: %sYes%s, ", green, normal);
+			else
+				printf("Files: %sNo%s, ", red, normal);
+			if(mg_check_feature(MG_FEATURES_TLS))
+				printf("TLS: %sYes%s, ", green, normal);
+			else
+				printf("TLS: %sNo%s, ", red, normal);
+			if(mg_check_feature(MG_FEATURES_CGI))
+				printf("CGI: %sYes%s, ", green, normal);
+			else
+				printf("CGI: %sNo%s, ", red, normal);
+			if(mg_check_feature(MG_FEATURES_IPV6))
+				printf("IPv6: %sYes%s, \n", green, normal);
+			else
+				printf("IPv6: %sNo%s, \n", red, normal);
+			if(mg_check_feature(MG_FEATURES_WEBSOCKET))
+				printf("                 WebSockets: %sYes%s, ", green, normal);
+			else
+				printf("                 WebSockets: %sNo%s, ", red, normal);
+			if(mg_check_feature(MG_FEATURES_SSJS))
+				printf("Server-side JavaScript: %sYes%s\n", green, normal);
+			else
+				printf("Server-side JavaScript: %sNo%s\n", red, normal);
+			if(mg_check_feature(MG_FEATURES_LUA))
+				printf("                 Lua: %sYes%s, ", green, normal);
+			else
+				printf("                 Lua: %sNo%s, ", red, normal);
+			if(mg_check_feature(MG_FEATURES_CACHE))
+				printf("Cache: %sYes%s, ", green, normal);
+			else
+				printf("Cache: %sNo%s, ", red, normal);
+			if(mg_check_feature(MG_FEATURES_STATS))
+				printf("Stats: %sYes%s, ", green, normal);
+			else
+				printf("Stats: %sNo%s, ", red, normal);
+			if(mg_check_feature(MG_FEATURES_COMPRESSION))
+				printf("Compression: %sYes%s\n", green, normal);
+			else
+				printf("Compression: %sNo%s\n", red, normal);
+			if(mg_check_feature(MG_FEATURES_HTTP2))
+				printf("                 HTTP2: %sYes%s, ", green, normal);
+			else
+				printf("                 HTTP2: %sNo%s, ", red, normal);
+			if(mg_check_feature(MG_FEATURES_X_DOMAIN_SOCKET))
+				printf("Unix domain sockets: %sYes%s\n", green, normal);
+			else
+				printf("Unix domain sockets: %sNo%s\n", red, normal);
+			printf("\n");
+			printf("****************************** %s%scJSON%s ********************************\n",
+			       yellow, bold, normal);
+			printf("Version:         %s%s%s%s\n", green, bold, cJSON_Version(), normal);
+			printf("\n");
 			exit(EXIT_SUCCESS);
 		}
 
@@ -557,12 +946,63 @@ void parse_args(int argc, char* argv[])
 			printf("\t%s--list-dhcp6%s        List known DHCPv6 config options\n\n", green, normal);
 
 			printf("%sDebugging and special use:%s\n", yellow, normal);
-			printf("\t%sd%s, %sdebug%s            Enter debugging mode\n", green, normal, green, normal);
+			printf("\t%sd%s, %sdebug%s            Enter debugging mode: Don't go into \n", green, normal, green, normal);
+			printf("\t                    daemon mode and verbose logging\n");
 			printf("\t%stest%s                Don't start pihole-FTL but instead\n", green, normal);
-			printf("\t                    quit immediately\n");
+			printf("\t                    process everything and quit immediately\n");
 			printf("\t%s-f%s, %sno-daemon%s       Don't go into daemon mode\n\n", green, normal, green, normal);
 
+			printf("%sConfig options:%s\n", yellow, normal);
+			printf("\t%s--config %skey%s        Get current value of config item %skey%s\n", green, blue, normal, blue, normal);
+			printf("\t%s--config %skey %svalue%s  Set new %svalue%s of config item %skey%s\n\n", green, blue, cyan, normal, cyan, normal, blue, normal);
+
+			printf("%sEmbedded GZIP un-/compressor:%s\n", yellow, normal);
+			printf("    A simple but fast in-memory gzip compressor\n\n");
+			printf("    Usage: %spihole-FTL --compress %sinfile %s[outfile]%s\n", green, cyan, purple, normal);
+			printf("    Usage: %spihole-FTL --uncompress %sinfile %s[outfile]%s\n\n", green, cyan, purple, normal);
+			printf("    - %sinfile%s is the file to be compressed.\n", cyan, normal);
+			printf("    - %s[outfile]%s is the optional target. If omitted, FTL will\n", purple, normal);
+			printf("      %s--compress%s:   use the %sinfile%s and append %s.gz%s at the end\n", green, normal, cyan, normal, cyan, normal);
+			printf("      %s--uncompress%s: use the %sinfile%s and remove %s.gz%s at the end\n\n", green, normal, cyan, normal, cyan, normal);
+
+			printf("%sTeleporter:%s\n", yellow, normal);
+			printf("\t%s--teleporter%s        Create a Teleporter archive in the\n", green, normal);
+			printf("\t                    current directory and print its name\n");
+			printf("\t%s--teleporter%s file%s   Import the Teleporter archive %sfile%s\n\n", green, cyan, normal, cyan, normal);
+
+			printf("%sTLS X.509 certificate generator:%s\n", yellow, normal);
+			printf("    Generate a self-signed certificate suitable for SSL/TLS\n");
+			printf("    and store it in %soutfile%s.\n\n", cyan, normal);
+			printf("    By default, this new certificate is based on the elliptic\n");
+			printf("    curve secp521r1. If the optional flag %s[rsa]%s is specified,\n", purple, normal);
+			printf("    an RSA (4096 bit) key will be generated instead.\n\n");
+			printf("    Usage: %spihole-FTL --gen-x509 %soutfile %s[rsa]%s\n\n", green, cyan, purple, normal);
+
+			printf("%sTLS X.509 certificate parser:%s\n", yellow, normal);
+			printf("    Parse the given X.509 certificate and optionally check if\n");
+			printf("    it matches a given domain. If no domain is given, only a\n");
+			printf("    human-readable output string is printed.\n\n");
+			printf("    If no certificate file is given, the one from the config\n");
+			printf("    is used (if applicable). If --read-x509-key is used, details\n");
+			printf("    about the private key are printed as well.\n\n");
+			printf("    Usage: %spihole-FTL --read-x509 %s[certfile] %s[domain]%s\n", green, cyan, purple, normal);
+			printf("    Usage: %spihole-FTL --read-x509-key %s[certfile] %s[domain]%s\n\n", green, cyan, purple, normal);
+
+			printf("%sGravity tools:%s\n", yellow, normal);
+			printf("    Check domains in a given file for validity using Pi-hole's\n");
+			printf("    gravity filters. The expected input format is one domain\n");
+			printf("    per line (no HOSTS lists, etc.)\n\n");
+			printf("    Usage: %spihole-FTL gravity checkList %sinfile%s\n\n", green, cyan, normal);
+
+			printf("%sIDN2 conversion:%s\n", yellow, normal);
+			printf("    Convert a given internationalized domain name (IDN) to\n");
+			printf("    punycode or vice versa.\n\n");
+			printf("    Encoding: %spihole-FTL idn2 %sdomain%s\n", green, cyan, normal);
+			printf("    Decoding: %spihole-FTL idn2 -d %spunycode%s\n\n", green, cyan, normal);
+
 			printf("%sOther:%s\n", yellow, normal);
+			printf("\t%sptr %sIP%s              Resolve IP address to hostname\n", green, cyan, normal);
+			printf("\t%ssha256sum %sfile%s      Calculate SHA256 checksum of a file\n", green, cyan, normal);
 			printf("\t%sdhcp-discover%s       Discover DHCP servers in the local\n", green, normal);
 			printf("\t                    network\n");
 			printf("\t%sarp-scan %s[-a/-x]%s    Use ARP to scan local network for\n", green, cyan, normal);
@@ -571,6 +1011,11 @@ void parse_args(int argc, char* argv[])
 			printf("\t                    interfaces\n");
 			printf("\t                    Append %s-x%s to force scan on all\n", cyan, normal);
 			printf("\t                    interfaces and scan 10x more often\n");
+			printf("\t%s--totp%s              Generate valid TOTP token for 2FA\n", green, normal);
+			printf("\t                    authentication (if enabled)\n");
+			printf("\t%s--perf%s              Run performance-tests based on the\n", green, normal);
+			printf("\t                    BALLOON password-hashing algorithm\n");
+			printf("\t%s--%s [OPTIONS]%s        Pass OPTIONS to internal dnsmasq resolver\n", green, cyan, normal);
 			printf("\t%s-h%s, %shelp%s            Display this help and exit\n\n", green, normal, green, normal);
 			exit(EXIT_SUCCESS);
 		}
@@ -582,12 +1027,6 @@ void parse_args(int argc, char* argv[])
 			exit(EXIT_SUCCESS);
 		}
 
-		// Return number of errors on this undocumented flag
-		if(strcmp(argv[i], "--check-structs") == 0)
-		{
-			exit(check_struct_sizes());
-		}
-
 		// Complain if invalid options have been found
 		if(!ok)
 		{
@@ -604,3 +1043,21 @@ void parse_args(int argc, char* argv[])
 		}
 	}
 }
+
+// defined in src/dnsmasq/option.c
+extern void reset_usage_indicator(void);
+// defined in src/log.h
+bool only_testing = false;
+void test_dnsmasq_options(int argc, const char *argv[])
+{
+	// Reset getopt before calling read_opts
+	optind = 0;
+
+	// Signal we don't want to jump back to FTL's main()
+	// but die after configuration parsing
+	only_testing = true;
+
+	// Call dnsmasq's option parser
+	reset_usage_indicator();
+	read_opts(argc, (char**)argv, NULL);
+}

src/args.h

@@ -25,7 +25,6 @@ const char *cli_bold(void) __attribute__ ((pure));
 const char *cli_normal(void) __attribute__ ((pure));
 const char *cli_over(void) __attribute__ ((pure));
 
-// defined in dnsmasq_interface.c
-int check_struct_sizes(void);
+void test_dnsmasq_options(int argc, const char *argv[]);
 
 #endif //ARGS_H

src/capabilities.c

@@ -14,12 +14,11 @@
 #undef __USE_XOPEN
 #include "FTL.h"
 #include "capabilities.h"
-#include "config.h"
+#include "config/config.h"
 #include "log.h"
 
 static const unsigned int capabilityIDs[]   = { CAP_CHOWN ,  CAP_DAC_OVERRIDE ,  CAP_DAC_READ_SEARCH ,  CAP_FOWNER ,  CAP_FSETID ,  CAP_KILL ,  CAP_SETGID ,  CAP_SETUID ,  CAP_SETPCAP ,  CAP_LINUX_IMMUTABLE ,  CAP_NET_BIND_SERVICE ,  CAP_NET_BROADCAST ,  CAP_NET_ADMIN ,  CAP_NET_RAW ,  CAP_IPC_LOCK ,  CAP_IPC_OWNER ,  CAP_SYS_MODULE ,  CAP_SYS_RAWIO ,  CAP_SYS_CHROOT ,  CAP_SYS_PTRACE ,  CAP_SYS_PACCT ,  CAP_SYS_ADMIN ,  CAP_SYS_BOOT ,  CAP_SYS_NICE ,  CAP_SYS_RESOURCE ,  CAP_SYS_TIME ,  CAP_SYS_TTY_CONFIG ,  CAP_MKNOD ,  CAP_LEASE ,  CAP_AUDIT_WRITE ,  CAP_AUDIT_CONTROL ,  CAP_SETFCAP };
 static const char*        capabilityNames[] = {"CAP_CHOWN", "CAP_DAC_OVERRIDE", "CAP_DAC_READ_SEARCH", "CAP_FOWNER", "CAP_FSETID", "CAP_KILL", "CAP_SETGID", "CAP_SETUID", "CAP_SETPCAP", "CAP_LINUX_IMMUTABLE", "CAP_NET_BIND_SERVICE", "CAP_NET_BROADCAST", "CAP_NET_ADMIN", "CAP_NET_RAW", "CAP_IPC_LOCK", "CAP_IPC_OWNER", "CAP_SYS_MODULE", "CAP_SYS_RAWIO", "CAP_SYS_CHROOT", "CAP_SYS_PTRACE", "CAP_SYS_PACCT", "CAP_SYS_ADMIN", "CAP_SYS_BOOT", "CAP_SYS_NICE", "CAP_SYS_RESOURCE", "CAP_SYS_TIME", "CAP_SYS_TTY_CONFIG", "CAP_MKNOD", "CAP_LEASE", "CAP_AUDIT_WRITE", "CAP_AUDIT_CONTROL", "CAP_SETFCAP"};
-static const unsigned int numCaps = sizeof(capabilityIDs) / sizeof(*capabilityIDs);
 
 bool check_capability(const unsigned int cap)
 {
@@ -93,53 +92,53 @@ bool check_capabilities(void)
 	data = calloc(sizeof(*data), capsize);
 	capget(hdr, data);
 
-	logg("***************************************");
-	logg("* Linux capability debugging enabled  *");
-	for(unsigned int i = 0u; i < numCaps; i++)
+	log_debug(DEBUG_CAPS, "***************************************");
+	log_debug(DEBUG_CAPS, "* Linux capability debugging enabled  *");
+	for(unsigned int i = 0u; i < ArraySize(capabilityIDs); i++)
 	{
 		const unsigned int capid = capabilityIDs[i];
-		logg("* %-24s (%02u) = %s%s%s *",
+		log_debug(DEBUG_CAPS, "* %-24s (%02u) = %s%s%s *",
 			capabilityNames[capid], capid,
 			((data->permitted   & (1 << capid)) ? "P":"-"),
 			((data->inheritable & (1 << capid)) ? "I":"-"),
 			((data->effective   & (1 << capid)) ? "E":"-"));
 	}
-	logg("***************************************");
+	log_debug(DEBUG_CAPS, "***************************************");
 
 	bool capabilities_okay = true;
 	if (!(data->permitted & (1 << CAP_NET_ADMIN)) ||
 	    !(data->effective & (1 << CAP_NET_ADMIN)))
 	{
 		// Needed for ARP-injection (used when we're the DHCP server)
-		logg("WARNING: Required Linux capability CAP_NET_ADMIN not available");
+		log_warn("Required Linux capability CAP_NET_ADMIN not available");
 		capabilities_okay = false;
 	}
 	if (!(data->permitted & (1 << CAP_NET_RAW)) ||
 	    !(data->effective & (1 << CAP_NET_RAW)))
 	{
 		// Needed for raw socket access (necessary for ICMP)
-		logg("WARNING: Required Linux capability CAP_NET_RAW not available");
+		log_warn("Required Linux capability CAP_NET_RAW not available");
 		capabilities_okay = false;
 	}
 	if (!(data->permitted & (1 << CAP_NET_BIND_SERVICE)) ||
 	    !(data->effective & (1 << CAP_NET_BIND_SERVICE)))
 	{
 		// Necessary for dynamic port binding
-		logg("WARNING: Required Linux capability CAP_NET_BIND_SERVICE not available");
+		log_warn("Required Linux capability CAP_NET_BIND_SERVICE not available");
 		capabilities_okay = false;
 	}
 	if (!(data->permitted & (1 << CAP_SYS_NICE)) ||
 	    !(data->effective & (1 << CAP_SYS_NICE)))
 	{
 		// Necessary for setting higher process priority through nice
-		logg("WARNING: Required Linux capability CAP_SYS_NICE not available");
+		log_warn("Required Linux capability CAP_SYS_NICE not available");
 		capabilities_okay = false;
 	}
 	if (!(data->permitted & (1 << CAP_CHOWN)) ||
 	    !(data->effective & (1 << CAP_CHOWN)))
 	{
 		// Necessary to chown required files that are owned by another user
-		logg("WARNING: Required Linux capability CAP_CHOWN not available");
+		log_warn("Required Linux capability CAP_CHOWN not available");
 		capabilities_okay = false;
 	}
 

src/capabilities.h

@@ -10,6 +10,8 @@
 #ifndef CAPABILITIES_H
 #define CAPABILITIES_H
 
+#include <linux/capability.h>
+
 bool check_capability(const unsigned int cap);
 bool check_capabilities(void);
 

src/config.h

@@ -1,114 +0,0 @@
-/* Pi-hole: A black hole for Internet advertisements
-*  (c) 2019 Pi-hole, LLC (https://pi-hole.net)
-*  Network-wide ad blocking via your own hardware.
-*
-*  FTL Engine
-*  FTL config file prototypes
-*
-*  This file is copyright under the latest version of the EUPL.
-*  Please see LICENSE file for your rights under this license. */
-#ifndef CONFIG_H
-#define CONFIG_H
-
-// enum privacy_level
-#include "enums.h"
-
-// typedef int16_t
-#include <sys/types.h>
-// typedef uni32_t
-#include <stdint.h>
-// struct in_addr, in6_addr
-#include <netinet/in.h>
-// type bool
-#include <stdbool.h>
-// type FILE
-#include <stdio.h>
-
-void init_config_mutex(void);
-void getLogFilePath(void);
-void read_FTLconf(void);
-void get_privacy_level(FILE *fp);
-void get_blocking_mode(FILE *fp);
-void read_debuging_settings(FILE *fp);
-
-// We do not use bitfields in here as this struct exists only once in memory.
-// Accessing bitfields may produce slightly more inefficient code on some
-// architectures (such as ARM) and savng a few bit of RAM but bloating up the
-// rest of the application each time these fields are accessed is bad.
-typedef struct {
-	bool socket_listenlocal :1;
-	bool analyze_AAAA :1;
-	bool resolveIPv6 :1;
-	bool resolveIPv4 :1;
-	bool ignore_localhost :1;
-	bool analyze_only_A_AAAA :1;
-	bool DBimport :1;
-	bool DBexport :1;
-	bool parse_arp_cache :1;
-	bool cname_inspection :1;
-	bool block_esni :1;
-	bool names_from_netdb :1;
-	bool edns0_ecs :1;
-	bool show_dnssec :1;
-	bool addr2line :1;
-	struct {
-		bool mozilla_canary :1;
-		bool icloud_private_relay :1;
-	} special_domains;
-	struct {
-		bool load :1;
-		unsigned char shmem;
-		unsigned char disk;
-	} check;
-	enum privacy_level privacylevel;
-	enum blocking_mode blockingmode;
-	enum refresh_hostnames refresh_hostnames;
-	enum busy_reply reply_when_busy;
-	enum ptr_type pihole_ptr;
-	int maxDBdays;
-	int port;
-	int maxlogage;
-	int dns_port;
-	unsigned int delay_startup;
-	unsigned int network_expire;
-	unsigned int block_ttl;
-	struct {
-		unsigned int count;
-		unsigned int interval;
-	} rate_limit;
-	enum debug_flags debug;
-	time_t DBinterval;
-	struct {
-		struct {
-			bool overwrite_v4 :1;
-			bool overwrite_v6 :1;
-			struct in_addr v4;
-			struct in6_addr v6;
-		} own_host;
-		struct {
-			bool overwrite_v4 :1;
-			bool overwrite_v6 :1;
-			struct in_addr v4;
-			struct in6_addr v6;
-		} ip_blocking;
-	} reply_addr;
-} ConfigStruct;
-
-typedef struct {
-	const char* conf;
-	const char* snapConf;
-	char* log;
-	char* pid;
-	char* port;
-	char* socketfile;
-	char* FTL_db;
-	char* gravity_db;
-	char* macvendor_db;
-	char* setupVars;
-	char* auditlist;
-} FTLFileNamesStruct;
-
-extern ConfigStruct config;
-extern FTLFileNamesStruct FTLfiles;
-
-#endif //CONFIG_H

src/config/CMakeLists.txt

@@ -0,0 +1,44 @@
+# Pi-hole: A black hole for Internet advertisements
+# (c) 2021 Pi-hole, LLC (https://pi-hole.net)
+# Network-wide ad blocking via your own hardware.
+#
+# FTL Engine
+# /src/config/CMakeList.txt
+#
+# This file is copyright under the latest version of the EUPL.
+# Please see LICENSE file for your rights under this license.
+
+set(sources
+        cli.c
+        cli.h
+        config.c
+        config.h
+        dnsmasq_config.c
+        dnsmasq_config.h
+        env.c
+        env.h
+        inotify.c
+        inotify.h
+        legacy_reader.c
+        legacy_reader.h
+        password.c
+        password.h
+        suggest.c
+        suggest.h
+        setupVars.c
+        setupVars.h
+        toml_writer.c
+        toml_writer.h
+        toml_reader.c
+        toml_reader.h
+        toml_helper.c
+        toml_helper.h
+        validator.c
+        validator.h
+        )
+
+add_library(config OBJECT ${sources})
+
+target_compile_options(config PRIVATE ${EXTRAWARN})
+target_include_directories(config PRIVATE ${PROJECT_SOURCE_DIR}/src)
+add_subdirectory(tomlc99)

src/config/cli.c

@@ -0,0 +1,583 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2023 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  CLI config routines
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+
+#include "FTL.h"
+#include "config/cli.h"
+#include "config/config.h"
+#include "config/toml_helper.h"
+#include "config/toml_writer.h"
+#include "config/dnsmasq_config.h"
+#include "log.h"
+#include "datastructure.h"
+// toml_table_t
+#include "tomlc99/toml.h"
+// hash_password()
+#include "config/password.h"
+// check_capability()
+#include "capabilities.h"
+// suggest_closest_conf_key()
+#include "config/suggest.h"
+
+enum exit_codes {
+	OKAY = 0,
+	FAIL = 1,
+	VALUE_INVALID = 2,
+	DNSMASQ_TEST_FAILED = 3,
+	KEY_UNKNOWN = 4,
+	ENV_VAR_FORCED = 5,
+} __attribute__((packed));
+
+// Read a TOML value from a table depending on its type
+static bool readStringValue(struct conf_item *conf_item, const char *value, struct config *newconf)
+{
+	if(conf_item == NULL || value == NULL)
+	{
+		log_debug(DEBUG_CONFIG, "readStringValue(%p, %p) called with invalid arguments, skipping",
+		          conf_item, value);
+		return false;
+	}
+	switch(conf_item->t)
+	{
+		case CONF_BOOL:
+		{
+			if(strcasecmp(value, "true") == 0 || strcasecmp(value, "yes") == 0)
+				conf_item->v.b = true;
+			else if(strcasecmp(value, "false") == 0 || strcasecmp(value, "no") == 0)
+				conf_item->v.b = false;
+			else
+			{
+				log_err("Config setting %s is invalid, allowed options are: [ true, false, yes, no ]", conf_item->k);
+				return false;
+			}
+			break;
+		}
+		case CONF_ALL_DEBUG_BOOL:
+		{
+			if(strcasecmp(value, "true") == 0 || strcasecmp(value, "yes") == 0)
+			{
+				set_all_debug(newconf, true);
+				conf_item->v.b = true;
+				set_debug_flags(newconf);
+			}
+			else if(strcasecmp(value, "false") == 0 || strcasecmp(value, "no") == 0)
+			{
+				set_all_debug(newconf, false);
+				conf_item->v.b = false;
+				set_debug_flags(newconf);
+			}
+			else
+			{
+				log_err("Config setting %s is invalid, allowed options are: [ true, false, yes, no ]", conf_item->k);
+				return false;
+			}
+			break;
+		}
+		case CONF_INT:
+		{
+			int val;
+			if(sscanf(value, "%i", &val) == 1)
+				conf_item->v.i = val;
+			else
+			{
+				log_err("Config setting %s is invalid, allowed options are: integer", conf_item->k);
+				return false;
+			}
+			break;
+		}
+		case CONF_UINT:
+		{
+			unsigned int val;
+			if(sscanf(value, "%u", &val) == 1)
+				conf_item->v.ui = val;
+			else
+			{
+				log_err("Config setting %s is invalid, allowed options are: unsigned integer", conf_item->k);
+				return false;
+			}
+			break;
+		}
+		case CONF_UINT16:
+		{
+			uint16_t val;
+			if(sscanf(value, "%hu", &val) == 1)
+				conf_item->v.ui = val;
+			else
+			{
+				log_err("Config setting %s is invalid, allowed options are: unsigned integer (16 bit)", conf_item->k);
+				return false;
+			}
+			break;
+		}
+		case CONF_LONG:
+		{
+			long val;
+			if(sscanf(value, "%li", &val) == 1)
+				conf_item->v.l = val;
+			else
+			{
+				log_err("Config setting %s is invalid, allowed options are: long integer", conf_item->k);
+				return false;
+			}
+			break;
+		}
+		case CONF_ULONG:
+		{
+			unsigned long val;
+			if(sscanf(value, "%lu", &val) == 1)
+				conf_item->v.ul = val;
+			else
+			{
+				log_err("Config setting %s is invalid, allowed options are: unsigned long integer", conf_item->k);
+				return false;
+			}
+			break;
+		}
+		case CONF_DOUBLE:
+		{
+			double val;
+			if(sscanf(value, "%lf", &val) == 1)
+				conf_item->v.d = val;
+			else
+			{
+				log_err("Config setting %s is invalid, allowed options are: double", conf_item->k);
+				return false;
+			}
+			break;
+		}
+		case CONF_STRING:
+		case CONF_STRING_ALLOCATED:
+		{
+			if(conf_item->t == CONF_STRING_ALLOCATED)
+					free(conf_item->v.s);
+			conf_item->v.s = strdup(value);
+			conf_item->t = CONF_STRING_ALLOCATED;
+			break;
+		}
+		case CONF_PASSWORD:
+		{
+			// Get pointer to pwhash instead of the password by
+			// decrementing the pointer by one. This is safe as we
+			// know that the pwhash is the immediately preceding
+			// item in the struct
+			conf_item--;
+
+			// Get password hash as allocated string (an empty string is hashed to an empty string)
+			char *pwhash = strlen(value) > 0 ? create_password(value) : strdup("");
+
+			// Verify that the password hash is either valid or empty
+			const enum password_result status = verify_password(value, pwhash, false);
+			if(status != PASSWORD_CORRECT && status != NO_PASSWORD_SET)
+			{
+				log_err("Failed to create password hash (verification failed), password remains unchanged");
+				free(pwhash);
+				return false;
+			}
+
+			// Free old password hash if it was allocated
+			if(conf_item->t == CONF_STRING_ALLOCATED)
+					free(conf_item->v.s);
+
+			// Store new password hash
+			conf_item->v.s = pwhash;
+			conf_item->t = CONF_STRING_ALLOCATED;
+			break;
+		}
+		case CONF_ENUM_PTR_TYPE:
+		{
+			const int ptr_type = get_ptr_type_val(value);
+			if(ptr_type != -1)
+				conf_item->v.ptr_type = ptr_type;
+			else
+			{
+				char *allowed = NULL;
+				CONFIG_ITEM_ARRAY(conf_item->a, allowed);
+				log_err("Config setting %s is invalid, allowed options are: %s", conf_item->k, allowed);
+				free(allowed);
+				return false;
+			}
+			break;
+		}
+		case CONF_ENUM_BUSY_TYPE:
+		{
+			const int busy_reply = get_busy_reply_val(value);
+			if(busy_reply != -1)
+				conf_item->v.busy_reply = busy_reply;
+			else
+			{
+				char *allowed = NULL;
+				CONFIG_ITEM_ARRAY(conf_item->a, allowed);
+				log_err("Config setting %s is invalid, allowed options are: %s", conf_item->k, allowed);
+				free(allowed);
+				return false;
+			}
+			break;
+		}
+		case CONF_ENUM_BLOCKING_MODE:
+		{
+			const int blocking_mode = get_blocking_mode_val(value);
+			if(blocking_mode != -1)
+				conf_item->v.blocking_mode = blocking_mode;
+			else
+			{
+				char *allowed = NULL;
+				CONFIG_ITEM_ARRAY(conf_item->a, allowed);
+				log_err("Config setting %s is invalid, allowed options are: %s", conf_item->k, allowed);
+				free(allowed);
+				return false;
+			}
+			break;
+		}
+		case CONF_ENUM_REFRESH_HOSTNAMES:
+		{
+			const int refresh_hostnames = get_refresh_hostnames_val(value);
+			if(refresh_hostnames != -1)
+				conf_item->v.refresh_hostnames = refresh_hostnames;
+			else
+			{
+				char *allowed = NULL;
+				CONFIG_ITEM_ARRAY(conf_item->a, allowed);
+				log_err("Config setting %s is invalid, allowed options are: %s", conf_item->k, allowed);
+				free(allowed);
+				return false;
+			}
+			break;
+		}
+		case CONF_ENUM_LISTENING_MODE:
+		{
+			const int listeningMode = get_listeningMode_val(value);
+			if(listeningMode != -1)
+				conf_item->v.listeningMode = listeningMode;
+			else
+			{
+				char *allowed = NULL;
+				CONFIG_ITEM_ARRAY(conf_item->a, allowed);
+				log_err("Config setting %s is invalid, allowed options are: %s", conf_item->k, allowed);
+				free(allowed);
+				return false;
+			}
+			break;
+		}
+		case CONF_ENUM_PRIVACY_LEVEL:
+		{
+			int val;
+			if(sscanf(value, "%i", &val) == 1 && val >= PRIVACY_SHOW_ALL && val <= PRIVACY_MAXIMUM)
+				conf_item->v.i = val;
+			else
+			{
+				log_err("Config setting %s is invalid, allowed options are: integer between %d and %d", conf_item->k, PRIVACY_SHOW_ALL, PRIVACY_MAXIMUM);
+				return false;
+			}
+			break;
+		}
+		case CONF_ENUM_WEB_THEME:
+		{
+			const int web_theme = get_web_theme_val(value);
+			if(web_theme != -1)
+				conf_item->v.web_theme = web_theme;
+			else
+			{
+				char *allowed = NULL;
+				CONFIG_ITEM_ARRAY(conf_item->a, allowed);
+				log_err("Config setting %s is invalid, allowed options are: %s", conf_item->k, allowed);
+				free(allowed);
+				return false;
+			}
+			break;
+		}
+		case CONF_ENUM_TEMP_UNIT:
+		{
+			const int temp_unit = get_temp_unit_val(value);
+			if(temp_unit != -1)
+				conf_item->v.temp_unit = temp_unit;
+			else
+			{
+				char *allowed = NULL;
+				CONFIG_ITEM_ARRAY(conf_item->a, allowed);
+				log_err("Config setting %s is invalid, allowed options are: %s", conf_item->k, allowed);
+				free(allowed);
+				return false;
+			}
+			break;
+		}
+		case CONF_STRUCT_IN_ADDR:
+		{
+			struct in_addr addr4 = { 0 };
+			if(strlen(value) == 0)
+			{
+				// Special case: empty string -> 0.0.0.0
+				conf_item->v.in_addr.s_addr = INADDR_ANY;
+			}
+			else if(inet_pton(AF_INET, value, &addr4))
+				memcpy(&conf_item->v.in_addr, &addr4, sizeof(addr4));
+			else
+			{
+				log_err("Config setting %s is invalid (%s), allowed options are: IPv4 address", conf_item->k, strerror(errno));
+				return false;
+			}
+			break;
+		}
+		case CONF_STRUCT_IN6_ADDR:
+		{
+			struct in6_addr addr6 = { 0 };
+			if(strlen(value) == 0)
+			{
+				// Special case: empty string -> ::
+				memcpy(&conf_item->v.in6_addr, &in6addr_any, sizeof(in6addr_any));
+			}
+			else if(inet_pton(AF_INET6, value, &addr6))
+				memcpy(&conf_item->v.in6_addr, &addr6, sizeof(addr6));
+			else
+			{
+				log_err("Config setting %s is invalid (%s), allowed options are: IPv6 address", conf_item->k, strerror(errno));
+				return false;
+			}
+			break;
+		}
+		case CONF_JSON_STRING_ARRAY:
+		{
+			const char *json_error = NULL;
+			cJSON *elem = cJSON_ParseWithOpts(value, &json_error, 0);
+			if(elem == NULL)
+			{
+				log_err("Config setting %s is invalid: not valid JSON, error at: %.20s", conf_item->k, json_error);
+				return false;
+			}
+			if(!cJSON_IsArray(elem))
+			{
+				log_err("Config setting %s is invalid: not a valid string array (example: [ \"a\", \"b\", \"c\" ])", conf_item->k);
+				return false;
+			}
+			const unsigned int elems = cJSON_GetArraySize(elem);
+			for(unsigned int i = 0; i < elems; i++)
+			{
+				const cJSON *item = cJSON_GetArrayItem(elem, i);
+				if(!cJSON_IsString(item))
+				{
+					log_err("Config setting %s is invalid: element with index %u is not a string", conf_item->k, i);
+					cJSON_Delete(elem);
+					return false;
+				}
+			}
+			// If we reach this point, all elements are valid
+			// Free previously allocated JSON array and replace with new
+			cJSON_Delete(conf_item->v.json);
+			conf_item->v.json = elem;
+			break;
+		}
+	}
+
+	return true;
+}
+
+int set_config_from_CLI(const char *key, const char *value)
+{
+	// Check if we are either
+	// - root, or
+	// - pihole with CAP_CHOWN capability on the pihole-FTL binary
+	const uid_t euid = geteuid();
+	const struct passwd *current_user = getpwuid(euid);
+	const bool is_root = euid == 0;
+	const bool is_pihole = current_user != NULL && strcmp(current_user->pw_name, "pihole") == 0;
+	const bool have_chown_cap = check_capability(CAP_CHOWN);
+	if(!is_root && !(is_pihole && have_chown_cap))
+	{
+		if(is_pihole)
+			printf("Permission error: CAP_CHOWN is missing on the binary\n");
+		else
+			printf("Permission error: User %s is not allowed to edit Pi-hole's config\n", current_user->pw_name);
+
+		printf("Please run this command using sudo\n\n");
+		return EXIT_FAILURE;
+	}
+
+	// Identify config option
+	struct config newconf;
+	duplicate_config(&newconf, &config);
+	struct conf_item *conf_item = NULL;
+	struct conf_item *new_item = NULL;
+	for(unsigned int i = 0; i < CONFIG_ELEMENTS; i++)
+	{
+		// Get pointer to (copied) memory location of this conf_item
+		struct conf_item *item = get_conf_item(&newconf, i);
+
+		if(strcmp(item->k, key) != 0)
+			continue;
+
+		if(item->f & FLAG_ENV_VAR)
+		{
+			log_err("Config option %s is read-only (set via environmental variable)", key);
+			free_config(&newconf);
+			return ENV_VAR_FORCED;
+		}
+
+		// This is the config option we are looking for
+		new_item = item;
+
+		// Also get pointer to memory location of this conf_item
+		conf_item = get_conf_item(&config, i);
+
+		// Break early
+		break;
+	}
+
+	// Check if we found the config option
+	if(new_item == NULL)
+	{
+		unsigned int N = 0;
+		char **matches = suggest_closest_conf_key(false, key, &N);
+		log_err("Unknown config option %s, did you mean:", key);
+		for(unsigned int i = 0; i < N; i++)
+			log_err(" - %s", matches[i]);
+		free(matches);
+
+		free_config(&newconf);
+		return KEY_UNKNOWN;
+	}
+
+	// Parse value
+	if(!readStringValue(new_item, value, &newconf))
+	{
+		free_config(&newconf);
+		return VALUE_INVALID;
+	}
+
+	// Check if value changed compared to current value
+	// Also check if this is the password config item change as this
+	// actually changed pwhash behind the scenes
+	if(!compare_config_item(conf_item->t, &new_item->v, &conf_item->v) ||
+	   conf_item->t == CONF_PASSWORD)
+	{
+		// Config item changed
+
+		// Validate new value(if validation function is defined)
+		if(new_item->c != NULL)
+		{
+			char errbuf[VALIDATOR_ERRBUF_LEN] = { 0 };
+			if(!new_item->c(&new_item->v, new_item->k, errbuf))
+			{
+				free_config(&newconf);
+				log_err("Invalid value: %s", errbuf);
+				return 3;
+			}
+		}
+
+		// Is this a dnsmasq option we need to check?
+		if(conf_item->f & FLAG_RESTART_FTL)
+		{
+			char errbuf[ERRBUF_SIZE] = { 0 };
+			if(!write_dnsmasq_config(&newconf, true, errbuf))
+			{
+				// Test failed
+				log_debug(DEBUG_CONFIG, "Config item %s: dnsmasq config test failed", conf_item->k);
+				free_config(&newconf);
+				return DNSMASQ_TEST_FAILED;
+			}
+		}
+		else if(conf_item == &config.dns.hosts)
+		{
+			// We need to rewrite the custom.list file but do not
+			// need to restart dnsmasq
+			write_custom_list();
+		}
+
+		// Install new configuration
+		replace_config(&newconf);
+
+		// Print value
+		writeTOMLvalue(stdout, -1, new_item->t, &new_item->v);
+	}
+	else
+	{
+		// No change
+		log_debug(DEBUG_CONFIG, "Config item %s: Unchanged", conf_item->k);
+		free_config(&newconf);
+
+		// Print value
+		writeTOMLvalue(stdout, -1, conf_item->t, &conf_item->v);
+	}
+
+	putchar('\n');
+	writeFTLtoml(false);
+	return OKAY;
+}
+
+int get_config_from_CLI(const char *key, const bool quiet)
+{
+	// Identify config option
+	struct conf_item *conf_item = NULL;
+
+	// We first loop over all config options to check if the one we are
+	// looking for is an exact match, use partial match otherwise
+	bool exactMatch = false;
+	for(unsigned int i = 0; i < CONFIG_ELEMENTS; i++)
+	{
+		// Get pointer to memory location of this conf_item
+		struct conf_item *item = get_conf_item(&config, i);
+
+		// Check if item.k is identical with key
+		if(strcmp(item->k, key) == 0)
+		{
+			exactMatch = true;
+			break;
+		}
+	}
+
+	// Loop over all config options again to find the one we are looking for
+	// (possibly partial match)
+	for(unsigned int i = 0; i < CONFIG_ELEMENTS; i++)
+	{
+		// Get pointer to memory location of this conf_item
+		struct conf_item *item = get_conf_item(&config, i);
+
+		// Check if item.k starts with key
+		if(key != NULL &&
+		   ((exactMatch && strcmp(item->k, key) != 0) ||
+		    (!exactMatch && strncmp(item->k, key, strlen(key)))))
+			continue;
+
+		// Skip write-only options
+		if(item->f & FLAG_WRITE_ONLY)
+			continue;
+
+		// This is the config option we are looking for
+		conf_item = item;
+
+		// Print key if this is not an exact match
+		if(key == NULL || strcmp(item->k, key) != 0)
+			printf("%s = ", item->k);
+
+		// Print value
+		if(conf_item-> f & FLAG_WRITE_ONLY)
+			puts("<write-only property>");
+		else
+			writeTOMLvalue(stdout, -1, conf_item->t, &conf_item->v);
+		putchar('\n');
+	}
+
+	// Check if we found the config option
+	if(conf_item == NULL)
+	{
+		unsigned int N = 0;
+		char **matches = suggest_closest_conf_key(false, key, &N);
+		log_err("Unknown config option %s, did you mean:", key);
+		for(unsigned int i = 0; i < N; i++)
+			log_err(" - %s", matches[i]);
+		free(matches);
+
+		return KEY_UNKNOWN;
+	}
+
+	// Use return status if this is a boolean value
+	// and we are in quiet mode
+	if(quiet && conf_item != NULL && conf_item->t == CONF_BOOL)
+		return conf_item->v.b ? OKAY : FAIL;
+
+	return OKAY;
+}

src/config/cli.h

@@ -0,0 +1,16 @@
+/* Pi-hole: A black hole for Internet advertisements
+*  (c) 2023 Pi-hole, LLC (https://pi-hole.net)
+*  Network-wide ad blocking via your own hardware.
+*
+*  FTL Engine
+*  FTL CLI config file prototypes
+*
+*  This file is copyright under the latest version of the EUPL.
+*  Please see LICENSE file for your rights under this license. */
+#ifndef CONFIG_CLI_H
+#define CONFIG_CLI_H
+
+int set_config_from_CLI(const char *key, const char *value);
+int get_config_from_CLI(const char *key, const bool quiet);
+
+#endif //CONFIG_CLI_H