From de53020961e688031670cbb474ca5ab220b032d0 Mon Sep 17 00:00:00 2001 From: Per Stark Date: Mon, 29 Jun 2026 19:48:38 +0200 Subject: [PATCH] dev: toolchain harmonization, additional checks and formatters --- .envrc | 4 - .github/workflows/release.yml | 8 +- .gitignore | 2 + .typos.toml | 6 + CHANGELOG.md | 2 + common/src/storage/types/system_settings.rs | 16 +- deny.toml | 5 +- evaluations/Cargo.toml | 1 + flake.lock | 23 +- flake.nix | 25 +- html-router/src/routes/ingestion/handlers.rs | 2 +- nix/build/cargo.nix | 220 +++++++++++ nix/build/default.nix | 71 ++++ nix/build/windows.nix | 152 +++++++ nix/dev-shell.nix | 172 -------- nix/dev/process-compose.yaml | 48 +++ nix/minne-lib.nix | 373 ------------------ nix/{ => modules}/checks.nix | 60 ++- nix/{ => modules}/context.nix | 9 +- nix/modules/dev-defaults.nix | 29 ++ nix/modules/dev-shell.nix | 254 ++++++++++++ nix/modules/formatter.nix | 42 ++ nix/{package.nix => modules/packages.nix} | 11 +- nix/nixos/minne.nix | 330 ++++++++++++++++ .../release.nix} | 1 - nix/packages/rust-toolchain.nix | 38 ++ .../surrealdb/binary.nix} | 9 +- nix/packages/surrealdb/default.nix | 5 + nix/{ => packages}/windows-cross.nix | 5 - nix/rust-toolchain.nix | 23 -- nix/tests/module-eval.nix | 128 ++++++ nix/tests/vm-smoke.nix | 44 +++ nix/versions.nix | 18 + nix/vm-smoke-test.nix | 72 ---- rust-toolchain.toml | 4 - rustfmt.toml | 1 - 36 files changed, 1514 insertions(+), 699 deletions(-) create mode 100644 .typos.toml create mode 100644 nix/build/cargo.nix create mode 100644 nix/build/default.nix create mode 100644 nix/build/windows.nix delete mode 100644 nix/dev-shell.nix create mode 100644 nix/dev/process-compose.yaml delete mode 100644 nix/minne-lib.nix rename nix/{ => modules}/checks.nix (54%) rename nix/{ => modules}/context.nix (70%) create mode 100644 nix/modules/dev-defaults.nix create mode 100644 nix/modules/dev-shell.nix create mode 100644 nix/modules/formatter.nix rename nix/{package.nix => modules/packages.nix} (84%) create mode 100644 nix/nixos/minne.nix rename nix/{minne-release.nix => packages/release.nix} (98%) create mode 100644 nix/packages/rust-toolchain.nix rename nix/{surrealdb-binary.nix => packages/surrealdb/binary.nix} (73%) create mode 100644 nix/packages/surrealdb/default.nix rename nix/{ => packages}/windows-cross.nix (94%) delete mode 100644 nix/rust-toolchain.nix create mode 100644 nix/tests/module-eval.nix create mode 100644 nix/tests/vm-smoke.nix create mode 100644 nix/versions.nix delete mode 100644 nix/vm-smoke-test.nix delete mode 100644 rust-toolchain.toml delete mode 100644 rustfmt.toml diff --git a/.envrc b/.envrc index 3add915..3550a30 100644 --- a/.envrc +++ b/.envrc @@ -1,5 +1 @@ -if ! has nix_direnv_manual_version; then - source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/3.0.6/direnvrc" "sha256-RYcUJaRS3fmx9EynmEn1o1Q4BAE_hNiBhqdwaFm6zSE=" -fi - use flake diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cd543dc..4bb1996 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,7 +65,7 @@ jobs: - uses: nix-community/cache-nix-action@v7 id: xwin-cache with: - primary-key: nix-Linux-xwin-${{ hashFiles('nix/windows-cross.nix', 'flake.lock') }} + primary-key: nix-Linux-xwin-${{ hashFiles('nix/packages/windows-cross.nix', 'flake.lock') }} restore-prefixes-first-match: nix-Linux-xwin- gc-max-store-size-linux: 8G @@ -275,9 +275,13 @@ jobs: ls -la exit 1 fi + ARGS=() + if [ -n "$PRERELEASE_FLAG" ]; then + ARGS+=("$PRERELEASE_FLAG") + fi gh release create "$TAG" \ --target "${{ github.sha }}" \ --title "minne $TAG" \ --notes-file "$RUNNER_TEMP/notes.txt" \ - $PRERELEASE_FLAG \ + "${ARGS[@]}" \ "${FILES[@]}" diff --git a/.gitignore b/.gitignore index 8ea34eb..3e3a641 100644 --- a/.gitignore +++ b/.gitignore @@ -3,11 +3,13 @@ .devenv node_modules config.yaml +.env.local target result data +.data database evaluations/cache/ diff --git a/.typos.toml b/.typos.toml new file mode 100644 index 0000000..6e08298 --- /dev/null +++ b/.typos.toml @@ -0,0 +1,6 @@ +[files] +# Vendored/minified and generated assets produce noise, not real typos. +extend-exclude = [ + "*.min.js", + "html-router/assets/style.css", +] diff --git a/CHANGELOG.md b/CHANGELOG.md index ae18b4a..1e0d1fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +Fix: added helpers to deserialize datetime correctly, fixes bug where cant change systemsettings on fresh deployments. +Infra: nix modularization using flake-parts, added lints and additional checks like statix and deadnix. ## 1.0.5 (2026-06-24) diff --git a/common/src/storage/types/system_settings.rs b/common/src/storage/types/system_settings.rs index 8abdab8..73ae689 100644 --- a/common/src/storage/types/system_settings.rs +++ b/common/src/storage/types/system_settings.rs @@ -2,7 +2,9 @@ use chrono::{DateTime, Utc}; use tracing::warn; use crate::utils::config::EmbeddingBackend; -use crate::utils::serde_helpers::deserialize_flexible_id; +use crate::utils::serde_helpers::{ + deserialize_flexible_id, deserialize_option_datetime, serialize_option_datetime, +}; use serde::{Deserialize, Serialize}; use crate::{error::AppError, storage::db::SurrealDbClient, storage::types::StoredObject}; @@ -26,13 +28,21 @@ pub struct SystemSettings { pub image_processing_prompt: String, pub voice_processing_model: String, /// When the maintainer last completed a scheduled `REBUILD INDEX` pass. - #[serde(default)] + #[serde( + default, + serialize_with = "serialize_option_datetime", + deserialize_with = "deserialize_option_datetime" + )] pub last_index_rebuild_at: Option>, /// Worker id holding the index-rebuild lease, if any. #[serde(default)] pub index_rebuild_lease_owner: Option, /// Lease expiry for in-flight scheduled index rebuilds. - #[serde(default)] + #[serde( + default, + serialize_with = "serialize_option_datetime", + deserialize_with = "deserialize_option_datetime" + )] pub index_rebuild_lease_expires_at: Option>, } diff --git a/deny.toml b/deny.toml index 40118f1..b936225 100644 --- a/deny.toml +++ b/deny.toml @@ -13,18 +13,21 @@ ignore = [] allow = [ "GPL-3.0", "AGPL-3.0", + "AGPL-3.0-or-later", "LGPL-2.1", "MIT", "Apache-2.0", + "Apache-2.0 WITH LLVM-exception", "MPL-2.0", "BSD-2-Clause", "BSD-3-Clause", "ISC", - "CC0-1.0", + "CC0-1.0", "Unlicense", "Zlib", "Unicode-3.0", "CDLA-Permissive-2.0", + "NCSA", ] confidence-threshold = 0.8 exceptions = [ diff --git a/evaluations/Cargo.toml b/evaluations/Cargo.toml index 7d96a72..3f02e24 100644 --- a/evaluations/Cargo.toml +++ b/evaluations/Cargo.toml @@ -2,6 +2,7 @@ name = "evaluations" version = "0.1.0" edition = "2024" +license = "AGPL-3.0-or-later" [lints] workspace = true diff --git a/flake.lock b/flake.lock index 3f9b26c..88a7408 100644 --- a/flake.lock +++ b/flake.lock @@ -150,7 +150,8 @@ "fenix": "fenix", "flake-parts": "flake-parts", "git-hooks": "git-hooks", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "treefmt-nix": "treefmt-nix" } }, "rust-analyzer-src": { @@ -169,6 +170,26 @@ "repo": "rust-analyzer", "type": "github" } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1780220602, + "narHash": "sha256-eynAfOmbmxJnkp7YewvCEbShNnnYJ9gLLqkzsYtBPeM=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "db947814a175b7ca6ded66e21383d938df01c227", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 09783c5..02e4daa 100644 --- a/flake.nix +++ b/flake.nix @@ -9,21 +9,23 @@ fenix.inputs.nixpkgs.follows = "nixpkgs"; git-hooks.url = "github:cachix/git-hooks.nix"; git-hooks.inputs.nixpkgs.follows = "nixpkgs"; + treefmt-nix.url = "github:numtide/treefmt-nix"; + treefmt-nix.inputs.nixpkgs.follows = "nixpkgs"; }; outputs = inputs@{ flake-parts, ... }: let - ortVersion = "1.23.2"; - toolchainFile = ./rust-toolchain.toml; - rustVersion = (builtins.fromTOML (builtins.readFile toolchainFile)).toolchain.channel; + versions = import ./nix/versions.nix; in flake-parts.lib.mkFlake { inherit inputs; } { imports = [ - ./nix/context.nix - ./nix/package.nix - ./nix/checks.nix - ./nix/dev-shell.nix + inputs.treefmt-nix.flakeModule + ./nix/modules/context.nix + ./nix/modules/packages.nix + ./nix/modules/checks.nix + ./nix/modules/formatter.nix + ./nix/modules/dev-shell.nix ]; systems = [ @@ -34,12 +36,17 @@ ]; _module.args = { - inherit ortVersion toolchainFile rustVersion; + inherit versions; }; flake = { + nixosModules = rec { + minne = import ./nix/nixos/minne.nix { inherit (inputs) self; }; + default = minne; + }; + lib = { - inherit ortVersion rustVersion; + inherit (versions) ortVersion rustVersion minneVersion; }; }; }; diff --git a/html-router/src/routes/ingestion/handlers.rs b/html-router/src/routes/ingestion/handlers.rs index 7791c46..bbe3148 100644 --- a/html-router/src/routes/ingestion/handlers.rs +++ b/html-router/src/routes/ingestion/handlers.rs @@ -232,7 +232,7 @@ pub async fn get_task_updates_stream( if updated_task.state.is_terminal() { // Send a specific event that HTMX uses to close the connection // Send a event to reload the recent content - // Send a event to remove the loading indicatior + // Send a event to remove the loading indicator let check_icon = state.templates.render("icons/check_icon.html", &context!{}).unwrap_or_else(|_| "Ok".to_string()); yield Ok(Event::default().event("stop_loading").data(check_icon)); yield Ok(Event::default().event("update_latest_content").data("Update latest content")); diff --git a/nix/build/cargo.nix b/nix/build/cargo.nix new file mode 100644 index 0000000..2e2d406 --- /dev/null +++ b/nix/build/cargo.nix @@ -0,0 +1,220 @@ +{ + inputs, + pkgs, + system, + src, + ortVersion, + versions, +}: +let + inherit (pkgs) lib; + inherit (inputs) crane fenix; + inherit (versions) + minneVersion + mozjsRelease + mozjsHashes + rustVersion + rustManifestSha256 + ; + + rustToolchain = import ../packages/rust-toolchain.nix { + inherit fenix system rustVersion; + manifestSha256 = rustManifestSha256; + }; + + craneLib = rustToolchain.mkCraneLib pkgs (crane.mkLib pkgs); + + libExt = if pkgs.stdenv.isDarwin then "dylib" else "so"; + + linuxRuntimeLibs = with pkgs; [ + libglvnd + stdenv.cc.cc.lib + zlib + fontconfig.lib + freetype + openssl.out + onnxruntime + ]; + + devGraphicsLibs = with pkgs; [ + wayland + libxkbcommon + pipewire + libglvnd + ]; + + wrapLinuxBinary = libExt: '' + wrapProgram $out/bin/main \ + --prefix LD_LIBRARY_PATH : ${lib.makeLibraryPath linuxRuntimeLibs} \ + --set ORT_DYLIB_PATH ${pkgs.onnxruntime}/lib/libonnxruntime.${libExt} + for b in worker server; do + if [ -x "$out/bin/$b" ]; then + wrapProgram $out/bin/$b \ + --prefix LD_LIBRARY_PATH : ${lib.makeLibraryPath linuxRuntimeLibs} \ + --set ORT_DYLIB_PATH ${pkgs.onnxruntime}/lib/libonnxruntime.${libExt} + fi + done + ''; + + mozjsTarget = + { + "x86_64-linux" = "x86_64-unknown-linux-gnu"; + "aarch64-linux" = "aarch64-unknown-linux-gnu"; + "aarch64-darwin" = "aarch64-apple-darwin"; + "x86_64-darwin" = "x86_64-apple-darwin"; + } + .${system} or (throw "mozjs prebuilt archive not configured for system ${system}"); + + mozjsArchive = pkgs.fetchurl { + url = "https://github.com/servo/mozjs/releases/download/${mozjsRelease}/libmozjs-${mozjsTarget}.tar.gz"; + hash = mozjsHashes.${mozjsTarget} or (throw "missing mozjs hash for ${mozjsTarget}"); + }; + + commonArgs = { + version = minneVersion; + src = lib.cleanSourceWith { + inherit src; + filter = + path: type: + craneLib.filterCargoSources path type + || lib.any (x: lib.hasPrefix (toString x) (toString path)) [ + (toString src + "/Cargo.lock") + (toString src + "/common/db") + (toString src + "/html-router/templates") + (toString src + "/html-router/assets") + ]; + }; + strictDeps = true; + + buildInputs = [ + pkgs.openssl + pkgs.libglvnd + pkgs.onnxruntime + pkgs.fontconfig + pkgs.libclang.lib + ]; + + nativeBuildInputs = [ + pkgs.pkg-config + pkgs.rustfmt + pkgs.makeWrapper + pkgs.python3 + pkgs.llvmPackages.llvm + pkgs.rustPlatform.bindgenHook + pkgs.stdenv.cc.cc.lib + ]; + + MOZJS_ARCHIVE = "${mozjsArchive}"; + env.LD_LIBRARY_PATH = lib.makeLibraryPath linuxRuntimeLibs; + }; + + cargoArtifacts = craneLib.buildDepsOnly ( + commonArgs + // { + pname = "minne"; + cargoExtraArgs = "--workspace"; + doCheck = false; + } + ); + + minne-pkg = + if pkgs.onnxruntime.version == ortVersion then + craneLib.buildPackage ( + commonArgs + // { + pname = "minne"; + version = minneVersion; + inherit cargoArtifacts; + doCheck = false; + doInstallCargoArtifacts = true; + + postInstall = + lib.optionalString pkgs.stdenv.isLinux (wrapLinuxBinary libExt) + + lib.optionalString pkgs.stdenv.isDarwin '' + for b in main worker server; do + if [ -x "$out/bin/$b" ]; then + wrapProgram $out/bin/$b \ + --set ORT_DYLIB_PATH ${pkgs.onnxruntime}/lib/libonnxruntime.${libExt} + fi + done + ''; + } + ) + else + throw "pkgs.onnxruntime.version (${pkgs.onnxruntime.version}) must match ortVersion (${ortVersion})"; + + targetTriple = pkgs.stdenv.hostPlatform.config; + + releaseCommonArgs = { + inherit minneVersion targetTriple; + bzip2 = pkgs.bzip2.out; + brotli = pkgs.brotli.lib; + srcRoot = src; + }; + + minne-release = + if pkgs.stdenv.isLinux then + pkgs.callPackage ../packages/release.nix ( + releaseCommonArgs + // { + platform = "linux"; + inherit minne-pkg; + } + ) + else if pkgs.stdenv.isDarwin then + pkgs.callPackage ../packages/release.nix ( + releaseCommonArgs + // { + platform = "darwin"; + inherit minne-pkg; + } + ) + else + null; + + dockerImage = pkgs.dockerTools.buildLayeredImage { + name = "minne"; + tag = minneVersion; + created = "now"; + + contents = [ + minne-pkg + pkgs.cacert + pkgs.bashInteractive + pkgs.libglvnd + pkgs.fontconfig.lib + pkgs.freetype + pkgs.stdenv.cc.cc.lib + ]; + + maxLayers = 25; + + config = { + Cmd = [ "${minne-pkg}/bin/main" ]; + Env = [ + "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-certificates.crt" + "ORT_DYLIB_PATH=${pkgs.onnxruntime}/lib/libonnxruntime.${libExt}" + ]; + ExposedPorts = { + "3000/tcp" = { }; + }; + User = "appuser"; + }; + }; +in +{ + inherit + lib + craneLib + rustToolchain + libExt + linuxRuntimeLibs + devGraphicsLibs + commonArgs + minneVersion + minne-pkg + minne-release + releaseCommonArgs + dockerImage + ; +} diff --git a/nix/build/default.nix b/nix/build/default.nix new file mode 100644 index 0000000..7d52d52 --- /dev/null +++ b/nix/build/default.nix @@ -0,0 +1,71 @@ +{ + inputs, + pkgs, + system, + src, + ortVersion, +}: +let + versions = import ../versions.nix; + + surrealdbPkg = import ../packages/surrealdb { inherit pkgs system; }; + + cargo = import ./cargo.nix { + inherit + inputs + pkgs + system + src + ortVersion + versions + ; + }; + + windows = import ./windows.nix { + inherit + pkgs + system + ortVersion + versions + ; + inherit (cargo) + craneLib + rustToolchain + commonArgs + minneVersion + releaseCommonArgs + ; + }; + + vmSmokeTest = + if system == "x86_64-linux" then + pkgs.callPackage ../tests/vm-smoke.nix { + inherit (cargo) minne-pkg; + inherit surrealdbPkg; + minneNixosModule = import ../nixos/minne.nix { }; + } + else + null; + + moduleEvalTest = + if pkgs.stdenv.isLinux then + pkgs.callPackage ../tests/module-eval.nix { + inherit (cargo) minne-pkg; + inherit surrealdbPkg; + minneNixosModule = import ../nixos/minne.nix { }; + inherit (inputs.nixpkgs.lib) nixosSystem; + } + else + null; +in +cargo +// windows +// { + inherit + src + ortVersion + surrealdbPkg + vmSmokeTest + moduleEvalTest + ; +} diff --git a/nix/build/windows.nix b/nix/build/windows.nix new file mode 100644 index 0000000..fbb031f --- /dev/null +++ b/nix/build/windows.nix @@ -0,0 +1,152 @@ +{ + pkgs, + system, + ortVersion, + versions, + craneLib, + rustToolchain, + commonArgs, + minneVersion, + releaseCommonArgs, +}: +let + inherit (versions) mozjsRelease mozjsArchiveWindowsHash ortArchiveWindowsHash; + inherit (rustToolchain) windowsTarget; + + mozjsArchiveWindows = pkgs.fetchurl { + url = "https://github.com/servo/mozjs/releases/download/${mozjsRelease}/libmozjs-${windowsTarget}.tar.gz"; + hash = mozjsArchiveWindowsHash; + }; + + ortArchiveWindows = pkgs.fetchurl { + url = "https://github.com/microsoft/onnxruntime/releases/download/v${ortVersion}/onnxruntime-win-x64-${ortVersion}.zip"; + hash = ortArchiveWindowsHash; + }; + + windowsCross = pkgs.callPackage ../packages/windows-cross.nix { }; + + inherit (windowsCross) clangClWrapper xwinCargoCache; + + msvcShim = pkgs.symlinkJoin { + name = "minne-msvc-shim"; + paths = [ + (pkgs.writeShellScriptBin "cl.exe" '' + exec ${clangClWrapper} "$@" + '') + (pkgs.writeShellScriptBin "ml64.exe" '' + exec ${pkgs.llvmPackages.llvm}/bin/llvm-ml64 "$@" + '') + ]; + }; + + xwinSetup = pkgs.writeShellScript "minne-xwin-setup" '' + set -eo pipefail + + cache=${xwinCargoCache} + crt="$cache/xwin/crt" + sdk="$cache/xwin/sdk" + + export PATH="${msvcShim}/bin:${pkgs.llvmPackages.clang-unwrapped}/bin:${pkgs.llvmPackages.lld}/bin:${pkgs.llvmPackages.llvm}/bin:$PATH" + export LD_LIBRARY_PATH="${pkgs.stdenv.cc.cc.lib}/lib''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + + export AR_x86_64_pc_windows_msvc=${pkgs.llvmPackages.llvm}/bin/llvm-lib + export BINDGEN_EXTRA_CLANG_ARGS_x86_64_pc_windows_msvc="-I$crt/include -I$sdk/include/ucrt -I$sdk/include/um -I$sdk/include/shared -I$sdk/include/winrt" + export CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER=${pkgs.llvmPackages.lld}/bin/lld-link + export CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_RUSTFLAGS="-C linker-flavor=lld-link -Lnative=$crt/lib/x86_64 -Lnative=$sdk/lib/um/x86_64 -Lnative=$sdk/lib/ucrt/x86_64" + export CC_x86_64_pc_windows_msvc=cl.exe + export CXX_x86_64_pc_windows_msvc=cl.exe + export REAL_CLANG_CL=${pkgs.llvmPackages.clang-unwrapped}/bin/clang-cl + export REAL_LLD_LINK=${pkgs.llvmPackages.lld}/bin/lld-link + + _imsvc="--target=x86_64-pc-windows-msvc -Wno-unused-command-line-argument -fuse-ld=lld-link /imsvc $crt/include /imsvc $sdk/include/ucrt /imsvc $sdk/include/um /imsvc $sdk/include/shared /imsvc $sdk/include/winrt" + export CFLAGS_x86_64_pc_windows_msvc="$_imsvc" + export CXXFLAGS_x86_64_pc_windows_msvc="$_imsvc /EHsc" + export CL_FLAGS="--target=x86_64-pc-windows-msvc -Wno-unused-command-line-argument -fuse-ld=lld-link /imsvc $crt/include /imsvc $sdk/include/ucrt /imsvc $sdk/include/um /imsvc $sdk/include/shared /imsvc $sdk/include/winrt" + + export CMAKE_GENERATOR=Ninja + export CMAKE_SYSTEM_NAME=Windows + export CMAKE_TOOLCHAIN_FILE_x86_64_pc_windows_msvc="$cache/cmake/clang-cl/x86_64-pc-windows-msvc-toolchain.cmake" + + export LIB="$crt/lib/x86_64;$sdk/lib/um/x86_64;$sdk/lib/ucrt/x86_64" + export RCFLAGS="-I$crt/include -I$sdk/include/ucrt -I$sdk/include/um -I$sdk/include/shared -I$sdk/include/winrt" + export TARGET_AR=${pkgs.llvmPackages.llvm}/bin/llvm-lib + export TARGET_CC=${pkgs.llvmPackages.clang-unwrapped}/bin/clang-cl + export TARGET_CXX=${pkgs.llvmPackages.clang-unwrapped}/bin/clang-cl + export WINEDEBUG=-all + ''; + + windowsCommonArgs = commonArgs // { + MOZJS_ARCHIVE = "${mozjsArchiveWindows}"; + CARGO_BUILD_TARGET = windowsTarget; + doIncludeCrossToolchainEnv = false; + env.CARGO_PROFILE = "dist"; + buildInputs = [ + pkgs.openssl + pkgs.fontconfig + pkgs.libclang.lib + ]; + nativeBuildInputs = commonArgs.nativeBuildInputs ++ [ + pkgs.llvmPackages.llvm + pkgs.llvmPackages.clang-unwrapped + pkgs.llvmPackages.lld + pkgs.stdenv.cc.cc.lib + ]; + }; + + windowsCargoArtifacts = + if system == "x86_64-linux" then + craneLib.buildDepsOnly ( + windowsCommonArgs + // { + pname = "minne"; + cargoExtraArgs = "--workspace"; + doCheck = false; + preBuild = "source ${xwinSetup}"; + } + ) + else + null; + + minne-pkg-windows = + if system == "x86_64-linux" then + craneLib.buildPackage ( + windowsCommonArgs + // { + pname = "minne-windows"; + version = minneVersion; + cargoArtifacts = windowsCargoArtifacts; + cargoExtraArgs = "--target ${windowsTarget} -p main --bin main --bin server --bin worker"; + doCheck = false; + doInstallCargoArtifacts = false; + preBuild = "source ${xwinSetup}"; + installPhaseCommand = '' + mkdir -p "$out/bin" + for b in main server worker; do + install -m 755 "target/${windowsTarget}/dist/$b.exe" "$out/bin/$b.exe" + done + ''; + } + ) + else + null; + + minne-release-windows = + if system == "x86_64-linux" then + pkgs.callPackage ../packages/release.nix ( + releaseCommonArgs + // { + platform = "windows"; + inherit minne-pkg-windows ortArchiveWindows; + targetTriple = windowsTarget; + } + ) + else + null; +in +{ + inherit + xwinCargoCache + minne-pkg-windows + minne-release-windows + ; +} diff --git a/nix/dev-shell.nix b/nix/dev-shell.nix deleted file mode 100644 index 6440ff6..0000000 --- a/nix/dev-shell.nix +++ /dev/null @@ -1,172 +0,0 @@ -# Local development shell, git hooks, and process-compose runner. -{ inputs, ... }: -{ - perSystem = - { - pkgs, - lib, - system, - minneCtx, - ... - }: - let - inherit (minneCtx) rustToolchain surrealdbPkg devGraphicsLibs; - - ortDylib = - if pkgs.stdenv.isDarwin then - "${pkgs.onnxruntime}/lib/libonnxruntime.dylib" - else - "${pkgs.onnxruntime}/lib/libonnxruntime.so"; - - processComposeFile = pkgs.writeText "minne-process-compose.yaml" '' - version: "0.5" - - environment: - - MINIO_ROOT_USER=minioadmin - - MINIO_ROOT_PASSWORD=minioadmin - - MINIO_REGION=us-east-1 - - processes: - surreal_db: - command: | - mkdir -p database - exec ${surrealdbPkg}/bin/surreal start \ - --bind 127.0.0.1:8000 \ - --log info \ - --user root_user \ - --pass root_password \ - rocksdb:database/database.db - availability: - restart: on_failure - - tailwind: - command: ${pkgs.tailwindcss_4}/bin/tailwindcss --cwd html-router -i app.css -o assets/style.css --watch=always - availability: - restart: on_failure - - minio: - command: | - mkdir -p .data/minio - exec ${pkgs.minio}/bin/minio server .data/minio \ - --address 127.0.0.1:19000 \ - --console-address 127.0.0.1:19001 - availability: - restart: on_failure - - minio_setup: - command: | - for _ in $(seq 1 30); do - if ${pkgs.minio-client}/bin/mc alias set local http://127.0.0.1:19000 minioadmin minioadmin 2>/dev/null; then - ${pkgs.minio-client}/bin/mc mb local/minne-tests --ignore-existing - exit 0 - fi - sleep 1 - done - echo "minio did not become ready" >&2 - exit 1 - depends_on: - minio: - condition: process_started - ''; - - processComposeRunner = pkgs.writeShellScriptBin "minne-dev-up" '' - set -euo pipefail - root="''${MINNE_ROOT:-$(git rev-parse --show-toplevel 2>/dev/null || pwd)}" - cd "$root" - exec ${pkgs.process-compose}/bin/process-compose up -f ${processComposeFile} "$@" - ''; - - moldFlags = if pkgs.stdenv.isLinux then "-C link-arg=-fuse-ld=mold" else ""; - - preCommitCheck = inputs.git-hooks.lib.${system}.run { - src = inputs.self; - hooks = { - rustfmt.enable = true; - clippy = { - enable = true; - settings.allFeatures = true; - }; - nixfmt.enable = true; - }; - tools = { - cargo = rustToolchain.toolchain; - clippy = rustToolchain.toolchain; - rustfmt = rustToolchain.toolchain; - nixfmt = pkgs.nixfmt; - }; - }; - - installGitHooks = '' - legacy="$(git rev-parse --git-path hooks/pre-commit.legacy 2>/dev/null || true)" - if [ -n "$legacy" ] && [ -f "$legacy" ]; then - rm -f "$legacy" - fi - # prek migration can leave core.hooksPath set; pre-commit refuses to install then. - hooks_path="$(git config --local --get core.hooksPath 2>/dev/null || true)" - if [ -n "$hooks_path" ]; then - git config --local --unset-all core.hooksPath - fi - ''; - - devPackages = [ - rustToolchain.toolchain - surrealdbPkg - processComposeRunner - pkgs.process-compose - pkgs.minio - pkgs.minio-client - pkgs.openssl - pkgs.nodejs - pkgs.watchman - pkgs.vscode-langservers-extracted - pkgs.cargo-xwin - pkgs.clang - pkgs.onnxruntime - pkgs.cargo-watch - pkgs.tailwindcss_4 - pkgs.python3 - pkgs.fontconfig - pkgs.fontconfig.dev - pkgs.libGL - pkgs.libGLU - pkgs.libclang - pkgs.mold - pkgs.nixfmt - ]; - - devEnv = { - NIX_CFLAGS_COMPILE = "-Wno-error=cpp"; - LIBCLANG_PATH = "${pkgs.libclang.lib}/lib"; - LD_LIBRARY_PATH = lib.makeLibraryPath devGraphicsLibs; - ORT_DYLIB_PATH = ortDylib; - S3_ENDPOINT = "http://127.0.0.1:19000"; - S3_BUCKET = "minne-tests"; - MINNE_TEST_S3_ENDPOINT = "http://127.0.0.1:19000"; - MINNE_TEST_S3_BUCKET = "minne-tests"; - RUSTFLAGS = moldFlags; - MINNE_ROOT = "${inputs.self}"; - }; - in - { - packages.process-compose-runner = processComposeRunner; - - apps.dev = { - type = "app"; - program = "${processComposeRunner}/bin/minne-dev-up"; - meta.description = "Start local dev services (SurrealDB, MinIO, Tailwind)"; - }; - - devShells.default = pkgs.mkShell { - packages = devPackages; - env = devEnv; - shellHook = '' - ${preCommitCheck.shellHook} - ${installGitHooks} - echo "Minne dev shell (fenix ${rustToolchain.rustVersion})" - echo " nix run .#dev # or: minne-dev-up" - echo " cargo test --workspace" - echo " nix flake check" - ''; - }; - }; -} diff --git a/nix/dev/process-compose.yaml b/nix/dev/process-compose.yaml new file mode 100644 index 0000000..ca67edd --- /dev/null +++ b/nix/dev/process-compose.yaml @@ -0,0 +1,48 @@ +version: "0.5" + +environment: + - MINIO_ROOT_USER=@MINIO_USER@ + - MINIO_ROOT_PASSWORD=@MINIO_PASSWORD@ + - MINIO_REGION=@MINIO_REGION@ + +processes: + surreal_db: + command: | + mkdir -p database + exec @SURREALDB@ start \ + --bind @SURREAL_BIND@ \ + --log info \ + --user @SURREAL_USER@ \ + --pass @SURREAL_PASS@ \ + rocksdb:database/database.db + availability: + restart: on_failure + + tailwind: + command: @TAILWIND@ --cwd html-router -i app.css -o assets/style.css --watch=always + availability: + restart: on_failure + + minio: + command: | + mkdir -p .data/minio + exec @MINIO@ server .data/minio \ + --address @MINIO_ADDRESS@ \ + --console-address 127.0.0.1:19001 + availability: + restart: on_failure + + minio_setup: + command: | + for _ in $(seq 1 30); do + if @MC@ alias set local @MINIO_ENDPOINT@ @MINIO_USER@ @MINIO_PASSWORD@ 2>/dev/null; then + @MC@ mb local/@MINIO_BUCKET@ --ignore-existing + exit 0 + fi + sleep 1 + done + echo "minio did not become ready" >&2 + exit 1 + depends_on: + minio: + condition: process_started diff --git a/nix/minne-lib.nix b/nix/minne-lib.nix deleted file mode 100644 index 0971fc3..0000000 --- a/nix/minne-lib.nix +++ /dev/null @@ -1,373 +0,0 @@ -# Shared build context for packages, checks, and the dev shell. -{ - inputs, - pkgs, - system, - src, - ortVersion, - toolchainFile, -}: -let - lib = pkgs.lib; - inherit (inputs) crane fenix; - - surrealdbPkg = - if system == "x86_64-linux" then pkgs.callPackage ./surrealdb-binary.nix { } else pkgs.surrealdb; - - rustToolchain = import ./rust-toolchain.nix { - inherit fenix system toolchainFile; - }; - - craneLib = rustToolchain.mkCraneLib pkgs (crane.mkLib pkgs); - - libExt = if pkgs.stdenv.isDarwin then "dylib" else "so"; - - linuxRuntimeLibs = with pkgs; [ - libglvnd - stdenv.cc.cc.lib - zlib - fontconfig.lib - freetype - openssl.out - onnxruntime - ]; - - devGraphicsLibs = with pkgs; [ - wayland - libxkbcommon - pipewire - libglvnd - ]; - - wrapLinuxBinary = libExt: '' - wrapProgram $out/bin/main \ - --prefix LD_LIBRARY_PATH : ${lib.makeLibraryPath linuxRuntimeLibs} \ - --set ORT_DYLIB_PATH ${pkgs.onnxruntime}/lib/libonnxruntime.${libExt} - for b in worker server; do - if [ -x "$out/bin/$b" ]; then - wrapProgram $out/bin/$b \ - --prefix LD_LIBRARY_PATH : ${lib.makeLibraryPath linuxRuntimeLibs} \ - --set ORT_DYLIB_PATH ${pkgs.onnxruntime}/lib/libonnxruntime.${libExt} - fi - done - ''; - - minneVersion = "1.0.5"; - mozjsRelease = "mozjs-sys-v140.10.1-0"; - - mozjsTarget = - { - "x86_64-linux" = "x86_64-unknown-linux-gnu"; - "aarch64-linux" = "aarch64-unknown-linux-gnu"; - "aarch64-darwin" = "aarch64-apple-darwin"; - "x86_64-darwin" = "x86_64-apple-darwin"; - } - .${system} or (throw "mozjs prebuilt archive not configured for system ${system}"); - - mozjsHashes = { - "x86_64-unknown-linux-gnu" = "sha256-e5kW8HTg6Hrd3sGgU9bqFNTTf7wJCChFOwKE3xyYT4Q="; - "aarch64-unknown-linux-gnu" = "sha256-VXrcktvjSH+14tO9Kzx+n9f/9ZQGAzfEsniiT+xKT6Q="; - "aarch64-apple-darwin" = "sha256-T3y73nVic6R60keUpmVRFe110Eh7AcE/VwZQWXRU9A0="; - "x86_64-apple-darwin" = "sha256-4v6f6c1OwYdg1FKnFfdLEsrRdyghcxup4gF7ioTZzm4="; - }; - - mozjsArchive = pkgs.fetchurl { - url = "https://github.com/servo/mozjs/releases/download/${mozjsRelease}/libmozjs-${mozjsTarget}.tar.gz"; - hash = mozjsHashes.${mozjsTarget} or (throw "missing mozjs hash for ${mozjsTarget}"); - }; - - commonArgs = { - version = minneVersion; - src = lib.cleanSourceWith { - inherit src; - filter = - path: type: - craneLib.filterCargoSources path type - || lib.any (x: lib.hasPrefix (toString x) (toString path)) [ - (toString src + "/Cargo.lock") - (toString src + "/common/db") - (toString src + "/html-router/templates") - (toString src + "/html-router/assets") - ]; - }; - strictDeps = true; - - buildInputs = [ - pkgs.openssl - pkgs.libglvnd - pkgs.onnxruntime - pkgs.fontconfig - pkgs.libclang.lib - ]; - - nativeBuildInputs = [ - pkgs.pkg-config - pkgs.rustfmt - pkgs.makeWrapper - pkgs.python3 - pkgs.llvmPackages.llvm - pkgs.rustPlatform.bindgenHook - pkgs.stdenv.cc.cc.lib - ]; - - MOZJS_ARCHIVE = "${mozjsArchive}"; - env.LD_LIBRARY_PATH = lib.makeLibraryPath linuxRuntimeLibs; - }; - - cargoArtifacts = craneLib.buildDepsOnly ( - commonArgs - // { - pname = "minne"; - cargoExtraArgs = "--workspace"; - doCheck = false; - } - ); - - minne-pkg = - if pkgs.onnxruntime.version == ortVersion then - craneLib.buildPackage ( - commonArgs - // { - pname = "minne"; - version = minneVersion; - inherit cargoArtifacts; - doCheck = false; - doInstallCargoArtifacts = true; - - postInstall = - lib.optionalString pkgs.stdenv.isLinux (wrapLinuxBinary libExt) - + lib.optionalString pkgs.stdenv.isDarwin '' - for b in main worker server; do - if [ -x "$out/bin/$b" ]; then - wrapProgram $out/bin/$b \ - --set ORT_DYLIB_PATH ${pkgs.onnxruntime}/lib/libonnxruntime.${libExt} - fi - done - ''; - } - ) - else - throw "pkgs.onnxruntime.version (${pkgs.onnxruntime.version}) must match ortVersion (${ortVersion})"; - - targetTriple = pkgs.stdenv.hostPlatform.config; - - releaseCommonArgs = { - inherit minneVersion targetTriple; - bzip2 = pkgs.bzip2.out; - brotli = pkgs.brotli.lib; - srcRoot = src; - }; - - minne-release = - if pkgs.stdenv.isLinux then - pkgs.callPackage ./minne-release.nix ( - releaseCommonArgs - // { - platform = "linux"; - inherit minne-pkg; - } - ) - else if pkgs.stdenv.isDarwin then - pkgs.callPackage ./minne-release.nix ( - releaseCommonArgs - // { - platform = "darwin"; - inherit minne-pkg; - } - ) - else - null; - - windowsTarget = rustToolchain.windowsTarget; - - mozjsArchiveWindows = pkgs.fetchurl { - url = "https://github.com/servo/mozjs/releases/download/${mozjsRelease}/libmozjs-${windowsTarget}.tar.gz"; - hash = "sha256-nEX55a4vZJGxlDMCea9TEee60HiNe/yQzXtUqMlaM3c="; - }; - - ortArchiveWindows = pkgs.fetchurl { - url = "https://github.com/microsoft/onnxruntime/releases/download/v${ortVersion}/onnxruntime-win-x64-${ortVersion}.zip"; - hash = "sha256-CzjfmvIYNOQec9YC2Q21ywbb0cphiUi48dZtYHrJ880="; - }; - - windowsCross = pkgs.callPackage ./windows-cross.nix { }; - - inherit (windowsCross) clangClWrapper xwinCargoCache; - - msvcShim = pkgs.symlinkJoin { - name = "minne-msvc-shim"; - paths = [ - (pkgs.writeShellScriptBin "cl.exe" '' - exec ${clangClWrapper} "$@" - '') - (pkgs.writeShellScriptBin "ml64.exe" '' - exec ${pkgs.llvmPackages.llvm}/bin/llvm-ml64 "$@" - '') - ]; - }; - - xwinSetup = pkgs.writeShellScript "minne-xwin-setup" '' - set -eo pipefail - - cache=${xwinCargoCache} - crt="$cache/xwin/crt" - sdk="$cache/xwin/sdk" - - export PATH="${msvcShim}/bin:${pkgs.llvmPackages.clang-unwrapped}/bin:${pkgs.llvmPackages.lld}/bin:${pkgs.llvmPackages.llvm}/bin:$PATH" - export LD_LIBRARY_PATH="${pkgs.stdenv.cc.cc.lib}/lib''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - - export AR_x86_64_pc_windows_msvc=${pkgs.llvmPackages.llvm}/bin/llvm-lib - export BINDGEN_EXTRA_CLANG_ARGS_x86_64_pc_windows_msvc="-I$crt/include -I$sdk/include/ucrt -I$sdk/include/um -I$sdk/include/shared -I$sdk/include/winrt" - export CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER=${pkgs.llvmPackages.lld}/bin/lld-link - export CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_RUSTFLAGS="-C linker-flavor=lld-link -Lnative=$crt/lib/x86_64 -Lnative=$sdk/lib/um/x86_64 -Lnative=$sdk/lib/ucrt/x86_64" - export CC_x86_64_pc_windows_msvc=cl.exe - export CXX_x86_64_pc_windows_msvc=cl.exe - export REAL_CLANG_CL=${pkgs.llvmPackages.clang-unwrapped}/bin/clang-cl - export REAL_LLD_LINK=${pkgs.llvmPackages.lld}/bin/lld-link - - _imsvc="--target=x86_64-pc-windows-msvc -Wno-unused-command-line-argument -fuse-ld=lld-link /imsvc $crt/include /imsvc $sdk/include/ucrt /imsvc $sdk/include/um /imsvc $sdk/include/shared /imsvc $sdk/include/winrt" - export CFLAGS_x86_64_pc_windows_msvc="$_imsvc" - export CXXFLAGS_x86_64_pc_windows_msvc="$_imsvc /EHsc" - export CL_FLAGS="--target=x86_64-pc-windows-msvc -Wno-unused-command-line-argument -fuse-ld=lld-link /imsvc $crt/include /imsvc $sdk/include/ucrt /imsvc $sdk/include/um /imsvc $sdk/include/shared /imsvc $sdk/include/winrt" - - export CMAKE_GENERATOR=Ninja - export CMAKE_SYSTEM_NAME=Windows - export CMAKE_TOOLCHAIN_FILE_x86_64_pc_windows_msvc="$cache/cmake/clang-cl/x86_64-pc-windows-msvc-toolchain.cmake" - - export LIB="$crt/lib/x86_64;$sdk/lib/um/x86_64;$sdk/lib/ucrt/x86_64" - export RCFLAGS="-I$crt/include -I$sdk/include/ucrt -I$sdk/include/um -I$sdk/include/shared -I$sdk/include/winrt" - export TARGET_AR=${pkgs.llvmPackages.llvm}/bin/llvm-lib - export TARGET_CC=${pkgs.llvmPackages.clang-unwrapped}/bin/clang-cl - export TARGET_CXX=${pkgs.llvmPackages.clang-unwrapped}/bin/clang-cl - export WINEDEBUG=-all - ''; - - windowsCommonArgs = commonArgs // { - MOZJS_ARCHIVE = "${mozjsArchiveWindows}"; - CARGO_BUILD_TARGET = windowsTarget; - doIncludeCrossToolchainEnv = false; - env.CARGO_PROFILE = "dist"; - buildInputs = [ - pkgs.openssl - pkgs.fontconfig - pkgs.libclang.lib - ]; - nativeBuildInputs = commonArgs.nativeBuildInputs ++ [ - pkgs.llvmPackages.llvm - pkgs.llvmPackages.clang-unwrapped - pkgs.llvmPackages.lld - pkgs.stdenv.cc.cc.lib - ]; - }; - - windowsCargoArtifacts = - if system == "x86_64-linux" then - craneLib.buildDepsOnly ( - windowsCommonArgs - // { - pname = "minne"; - cargoExtraArgs = "--workspace"; - doCheck = false; - preBuild = "source ${xwinSetup}"; - } - ) - else - null; - - minne-pkg-windows = - if system == "x86_64-linux" then - craneLib.buildPackage ( - windowsCommonArgs - // { - pname = "minne-windows"; - version = minneVersion; - cargoArtifacts = windowsCargoArtifacts; - cargoExtraArgs = "--target ${windowsTarget} -p main --bin main --bin server --bin worker"; - doCheck = false; - doInstallCargoArtifacts = false; - preBuild = "source ${xwinSetup}"; - installPhaseCommand = '' - mkdir -p "$out/bin" - for b in main server worker; do - install -m 755 "target/${windowsTarget}/dist/$b.exe" "$out/bin/$b.exe" - done - ''; - } - ) - else - null; - - minne-release-windows = - if system == "x86_64-linux" then - pkgs.callPackage ./minne-release.nix ( - releaseCommonArgs - // { - platform = "windows"; - inherit minne-pkg-windows ortArchiveWindows; - targetTriple = windowsTarget; - } - ) - else - null; - - dockerImage = pkgs.dockerTools.buildLayeredImage { - name = "minne"; - tag = minneVersion; - created = "now"; - - contents = [ - minne-pkg - pkgs.cacert - pkgs.bashInteractive - pkgs.libglvnd - pkgs.fontconfig.lib - pkgs.freetype - pkgs.stdenv.cc.cc.lib - ]; - - maxLayers = 25; - - config = { - Cmd = [ "${minne-pkg}/bin/main" ]; - Env = [ - "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-certificates.crt" - "ORT_DYLIB_PATH=${pkgs.onnxruntime}/lib/libonnxruntime.${libExt}" - ]; - ExposedPorts = { - "3000/tcp" = { }; - }; - User = "appuser"; - }; - }; - - vmSmokeTest = - if system == "x86_64-linux" then - pkgs.callPackage ./vm-smoke-test.nix { - inherit minne-pkg; - surrealdb = surrealdbPkg; - } - else - null; -in -{ - inherit - src - ortVersion - lib - libExt - craneLib - rustToolchain - surrealdbPkg - linuxRuntimeLibs - devGraphicsLibs - commonArgs - minneVersion - minne-pkg - minne-pkg-windows - minne-release - minne-release-windows - dockerImage - vmSmokeTest - xwinCargoCache - ; -} diff --git a/nix/checks.nix b/nix/modules/checks.nix similarity index 54% rename from nix/checks.nix rename to nix/modules/checks.nix index 3e303af..762b163 100644 --- a/nix/checks.nix +++ b/nix/modules/checks.nix @@ -1,8 +1,10 @@ -# Flake checks: version gates, clippy, tests, formatting, VM smoke. { - ortVersion, + versions, ... }: +let + inherit (versions) ortVersion; +in { perSystem = { @@ -17,8 +19,8 @@ rustToolchain commonArgs minne-pkg - minneVersion vmSmokeTest + moduleEvalTest src ; in @@ -27,7 +29,7 @@ ortVersion = pkgs.runCommand "ort-version-check" { } '' if [ "${pkgs.onnxruntime.version}" != "${ortVersion}" ]; then echo "pkgs.onnxruntime.version is ${pkgs.onnxruntime.version}, but flake pins ${ortVersion}" >&2 - echo "Update ortVersion in flake.nix or wait for nixpkgs to catch up." >&2 + echo "Update nix/versions.nix or wait for nixpkgs to catch up." >&2 exit 1 fi touch $out @@ -43,7 +45,7 @@ actual="$(${rustToolchain.toolchain}/bin/rustc --version | awk '{print $2}')" if [ "$actual" != "$expected" ]; then echo "rustc version mismatch: expected $expected, got $actual" >&2 - echo "Update rust-toolchain.toml and rebuild the fenix toolchain." >&2 + echo "Update rustVersion in nix/versions.nix and rebuild the fenix toolchain." >&2 exit 1 fi touch $out @@ -69,14 +71,52 @@ } ); - minne-fmt = craneLib.cargoFmt { - pname = "minne-fmt"; - version = minneVersion; - src = craneLib.cleanCargoSource src; - }; + minne-deny = craneLib.cargoDeny ( + commonArgs + // { + cargoArtifacts = minne-pkg; + pname = "minne"; + cargoDenyChecks = "bans licenses sources"; + } + ); + + deadnix = pkgs.runCommand "deadnix-check" { nativeBuildInputs = [ pkgs.deadnix ]; } '' + deadnix --fail ${src}/nix ${src}/flake.nix + touch $out + ''; + + statix = pkgs.runCommand "statix-check" { nativeBuildInputs = [ pkgs.statix ]; } '' + statix check ${src} + touch $out + ''; + + actionlint = + pkgs.runCommand "actionlint-check" + { + nativeBuildInputs = [ + pkgs.actionlint + pkgs.shellcheck + ]; + } + '' + cd ${src} + shopt -s nullglob + files=(.github/workflows/*.yml .github/workflows/*.yaml) + actionlint "''${files[@]}" + touch $out + ''; + + typos = pkgs.runCommand "typos-check" { nativeBuildInputs = [ pkgs.typos ]; } '' + cd ${src} + typos + touch $out + ''; } // lib.optionalAttrs (vmSmokeTest != null) { minne-vm-smoke = vmSmokeTest; + } + // lib.optionalAttrs (moduleEvalTest != null) { + minne-module-eval = moduleEvalTest; }; }; } diff --git a/nix/context.nix b/nix/modules/context.nix similarity index 70% rename from nix/context.nix rename to nix/modules/context.nix index cebcded..2b22271 100644 --- a/nix/context.nix +++ b/nix/modules/context.nix @@ -1,14 +1,14 @@ -# Evaluates shared build context once per system for downstream flake-parts modules. { inputs, - ortVersion, - toolchainFile, + versions, ... }: { perSystem = { system, ... }: let + inherit (versions) ortVersion; + pkgs = import inputs.nixpkgs { inherit system; config = { @@ -19,13 +19,12 @@ in { _module.args.pkgs = pkgs; - _module.args.minneCtx = import ./minne-lib.nix { + _module.args.minneCtx = import ../build/default.nix { inherit inputs pkgs system ortVersion - toolchainFile ; src = inputs.self; }; diff --git a/nix/modules/dev-defaults.nix b/nix/modules/dev-defaults.nix new file mode 100644 index 0000000..f56accc --- /dev/null +++ b/nix/modules/dev-defaults.nix @@ -0,0 +1,29 @@ +{ + surreal = { + host = "127.0.0.1"; + port = 8000; + user = "root_user"; + pass = "root_password"; + namespace = "minne_ns"; + database = "minne_db"; + }; + + minio = { + endpoint = "http://127.0.0.1:19000"; + accessKey = "minioadmin"; + secretKey = "minioadmin"; + bucket = "minne-tests"; + region = "us-east-1"; + }; + + app = { + httpPort = 3009; + dataDir = "./data"; + # Replace in `.env.local` for real LLM use. + openaiApiKey = "local-dev-placeholder"; + embeddingBackend = "fastembed"; + pdfIngestMode = "classic"; + storage = "local"; + rustLog = "info"; + }; +} diff --git a/nix/modules/dev-shell.nix b/nix/modules/dev-shell.nix new file mode 100644 index 0000000..d227452 --- /dev/null +++ b/nix/modules/dev-shell.nix @@ -0,0 +1,254 @@ +{ inputs, ... }: +{ + perSystem = + { + config, + pkgs, + lib, + system, + minneCtx, + ... + }: + let + inherit (minneCtx) rustToolchain surrealdbPkg devGraphicsLibs; + + devDefaults = import ./dev-defaults.nix; + inherit (devDefaults) surreal minio app; + + ortDylib = + if pkgs.stdenv.isDarwin then + "${pkgs.onnxruntime}/lib/libonnxruntime.dylib" + else + "${pkgs.onnxruntime}/lib/libonnxruntime.so"; + + processComposeTemplate = builtins.readFile ../dev/process-compose.yaml; + + minioAddress = lib.removePrefix "http://" minio.endpoint; + + processComposeFile = pkgs.writeText "minne-process-compose.yaml" ( + lib.replaceStrings + [ + "@SURREALDB@" + "@TAILWIND@" + "@MINIO@" + "@MC@" + "@SURREAL_BIND@" + "@SURREAL_USER@" + "@SURREAL_PASS@" + "@MINIO_ADDRESS@" + "@MINIO_ENDPOINT@" + "@MINIO_USER@" + "@MINIO_PASSWORD@" + "@MINIO_REGION@" + "@MINIO_BUCKET@" + ] + [ + "${surrealdbPkg}/bin/surreal" + "${pkgs.tailwindcss_4}/bin/tailwindcss" + "${pkgs.minio}/bin/minio" + "${pkgs.minio-client}/bin/mc" + "${surreal.host}:${toString surreal.port}" + surreal.user + surreal.pass + minioAddress + minio.endpoint + minio.accessKey + minio.secretKey + minio.region + minio.bucket + ] + processComposeTemplate + ); + + # Shared project-local socket so `up` and `down` always target the same + # process-compose instance regardless of the caller's working directory. + processComposeSocket = ".data/process-compose.sock"; + + # Resolve the writable git checkout root; never the read-only flake store. + resolveRoot = name: '' + root="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" + if [ ! -w "$root" ]; then + echo "${name}: workspace is not writable: $root" >&2 + exit 1 + fi + cd "$root" + ''; + + processComposeRunner = pkgs.writeShellScriptBin "minne-dev-up" '' + set -euo pipefail + ${resolveRoot "minne-dev-up"} + mkdir -p .data/minio database html-router/assets + exec ${pkgs.process-compose}/bin/process-compose up \ + -u ${processComposeSocket} -U \ + -f ${processComposeFile} "$@" + ''; + + processComposeDownRunner = pkgs.writeShellScriptBin "minne-dev-down" '' + set -euo pipefail + ${resolveRoot "minne-dev-down"} + exec ${pkgs.process-compose}/bin/process-compose down \ + -u ${processComposeSocket} -U "$@" + ''; + + moldFlags = if pkgs.stdenv.isLinux then "-C link-arg=-fuse-ld=mold" else ""; + + cargoDenyRunner = pkgs.writeShellScriptBin "cargo-deny-check" '' + export PATH="${ + lib.makeBinPath [ + rustToolchain.toolchain + pkgs.cargo-deny + ] + }:$PATH" + exec cargo deny check licenses bans sources + ''; + + preCommitCheck = inputs.git-hooks.lib.${system}.run { + src = inputs.self; + hooks = { + rustfmt.enable = true; + clippy = { + enable = true; + settings.allFeatures = true; + }; + nixfmt.enable = true; + deadnix.enable = true; + statix.enable = true; + actionlint.enable = true; + typos.enable = true; + cargo-deny = { + enable = true; + name = "cargo-deny"; + description = "Check dependency licenses, bans, and sources"; + entry = "${cargoDenyRunner}/bin/cargo-deny-check"; + files = "(Cargo\\.(toml|lock)|deny\\.toml)$"; + pass_filenames = false; + }; + }; + tools = { + cargo = rustToolchain.toolchain; + clippy = rustToolchain.toolchain; + rustfmt = rustToolchain.toolchain; + inherit (pkgs) nixfmt; + }; + }; + + installGitHooks = '' + legacy="$(git rev-parse --git-path hooks/pre-commit.legacy 2>/dev/null || true)" + if [ -n "$legacy" ] && [ -f "$legacy" ]; then + rm -f "$legacy" + fi + # prek migration can leave core.hooksPath set; pre-commit refuses to install then. + hooks_path="$(git config --local --get core.hooksPath 2>/dev/null || true)" + if [ -n "$hooks_path" ]; then + git config --local --unset-all core.hooksPath + fi + ''; + + devPackages = [ + config.treefmt.build.wrapper + rustToolchain.toolchain + surrealdbPkg + processComposeRunner + processComposeDownRunner + pkgs.process-compose + pkgs.minio + pkgs.minio-client + pkgs.nodejs + pkgs.watchman + pkgs.vscode-langservers-extracted + pkgs.cargo-xwin + pkgs.clang + pkgs.onnxruntime + pkgs.cargo-watch + pkgs.tailwindcss_4 + pkgs.python3 + pkgs.fontconfig + pkgs.fontconfig.dev + pkgs.libGL + pkgs.libGLU + pkgs.libclang + pkgs.mold + pkgs.nixfmt + pkgs.deadnix + pkgs.statix + pkgs.actionlint + pkgs.typos + pkgs.cargo-deny + cargoDenyRunner + ]; + + loadLocalEnv = '' + root="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" + local_env="$root/.env.local" + if [ -f "$local_env" ]; then + set -a + # shellcheck source=/dev/null + source "$local_env" + set +a + fi + ''; + + devEnv = { + NIX_CFLAGS_COMPILE = "-Wno-error=cpp"; + LIBCLANG_PATH = "${pkgs.libclang.lib}/lib"; + LD_LIBRARY_PATH = lib.makeLibraryPath devGraphicsLibs; + ORT_DYLIB_PATH = ortDylib; + RUSTFLAGS = moldFlags; + + # Minne app — matches `nix run .#dev` services; override via `.env.local`. + OPENAI_API_KEY = app.openaiApiKey; + SURREALDB_ADDRESS = "ws://${surreal.host}:${toString surreal.port}"; + SURREALDB_USERNAME = surreal.user; + SURREALDB_PASSWORD = surreal.pass; + SURREALDB_NAMESPACE = surreal.namespace; + SURREALDB_DATABASE = surreal.database; + HTTP_PORT = toString app.httpPort; + DATA_DIR = app.dataDir; + EMBEDDING_BACKEND = app.embeddingBackend; + PDF_INGEST_MODE = app.pdfIngestMode; + STORAGE = app.storage; + RUST_LOG = app.rustLog; + + S3_ENDPOINT = minio.endpoint; + S3_BUCKET = minio.bucket; + S3_REGION = minio.region; + AWS_ACCESS_KEY_ID = minio.accessKey; + AWS_SECRET_ACCESS_KEY = minio.secretKey; + MINNE_TEST_S3_ENDPOINT = minio.endpoint; + MINNE_TEST_S3_BUCKET = minio.bucket; + }; + in + { + apps.dev = { + type = "app"; + program = "${processComposeRunner}/bin/minne-dev-up"; + meta.description = "Start local dev services (SurrealDB, MinIO, Tailwind)"; + }; + + apps.dev-down = { + type = "app"; + program = "${processComposeDownRunner}/bin/minne-dev-down"; + meta.description = "Stop local dev services started by minne-dev-up"; + }; + + devShells.default = pkgs.mkShell { + packages = devPackages; + nativeBuildInputs = [ pkgs.pkg-config ]; + buildInputs = [ pkgs.openssl ]; + env = devEnv; + shellHook = '' + ${preCommitCheck.shellHook} + ${installGitHooks} + ${loadLocalEnv} + echo "Minne dev shell (fenix ${rustToolchain.rustVersion})" + echo " nix run .#dev # or: minne-dev-up" + echo " nix run .#dev-down # or: minne-dev-down" + echo " nix fmt # rustfmt + nixfmt" + echo " cargo run -p main # env from dev shell + optional .env.local" + echo " overrides: cp .env.local.example .env.local" + echo " cargo test --workspace" + echo " nix flake check" + ''; + }; + }; +} diff --git a/nix/modules/formatter.nix b/nix/modules/formatter.nix new file mode 100644 index 0000000..9755482 --- /dev/null +++ b/nix/modules/formatter.nix @@ -0,0 +1,42 @@ +_: { + perSystem = + { + config, + minneCtx, + ... + }: + let + inherit (minneCtx) rustToolchain; + in + { + treefmt = { + projectRootFile = "flake.nix"; + + programs = { + nixfmt.enable = true; + rustfmt = { + enable = true; + edition = "2024"; + package = rustToolchain.toolchain; + }; + }; + + settings.global.excludes = [ + "**/.direnv/**" + "**/.data/**" + "**/database/**" + "**/evaluations/cache/**" + "**/evaluations/reports/**" + "**/html-router/node_modules/**" + "**/result/**" + "**/target/**" + ]; + }; + + apps.fmt = { + type = "app"; + program = "${config.treefmt.build.wrapper}/bin/treefmt"; + meta.description = "Format the repository (rustfmt, nixfmt)"; + }; + }; +} diff --git a/nix/package.nix b/nix/modules/packages.nix similarity index 84% rename from nix/package.nix rename to nix/modules/packages.nix index 4bbd189..abc05ec 100644 --- a/nix/package.nix +++ b/nix/modules/packages.nix @@ -1,9 +1,6 @@ -# Crane packages, release archives, and flake apps. -{ ... }: -{ +_: { perSystem = { - pkgs, lib, minneCtx, ... @@ -11,7 +8,6 @@ let inherit (minneCtx) minne-pkg - minne-pkg-windows minne-release minne-release-windows dockerImage @@ -24,11 +20,10 @@ default = minne-pkg; } // lib.optionalAttrs (minne-release != null) { - minne-release = minne-release; + inherit minne-release; } // lib.optionalAttrs (minne-release-windows != null) { - inherit xwinCargoCache; - minne-release-windows = minne-release-windows; + inherit xwinCargoCache minne-release-windows; }; apps = { diff --git a/nix/nixos/minne.nix b/nix/nixos/minne.nix new file mode 100644 index 0000000..c107f4c --- /dev/null +++ b/nix/nixos/minne.nix @@ -0,0 +1,330 @@ +{ + self ? null, +}: +{ + config, + lib, + pkgs, + ... +}: +let + cfg = config.services.minne; + + surrealAddress = "ws://${cfg.surrealdb.host}:${toString cfg.surrealdb.port}"; + + surrealUnits = + lib.optional cfg.surrealdb.enable "surrealdb.service" + ++ cfg.surrealdb.after + ++ cfg.surrealdb.requires; + + minneAfter = [ "network.target" ] ++ surrealUnits; + minneRequires = surrealUnits; + + surrealAuthArgs = + if cfg.surrealdb.username != null then + "--user ${cfg.surrealdb.username} --pass ${cfg.surrealdb.password}" + else + ""; + + baseEnvironment = { + SURREALDB_ADDRESS = surrealAddress; + HTTP_PORT = toString cfg.port; + RUST_LOG = cfg.logLevel; + DATA_DIR = cfg.dataDir; + SURREALDB_NAMESPACE = cfg.surrealdb.namespace; + SURREALDB_DATABASE = cfg.surrealdb.database; + } + // lib.optionalAttrs (cfg.surrealdb.username != null) { + SURREALDB_USERNAME = cfg.surrealdb.username; + SURREALDB_PASSWORD = cfg.surrealdb.password; + } + // cfg.environment; + + mkService = + { description, bin }: + { + inherit description; + wantedBy = [ "multi-user.target" ]; + after = minneAfter; + requires = minneRequires; + environment = baseEnvironment; + serviceConfig = { + Type = "simple"; + User = cfg.user; + Group = cfg.group; + WorkingDirectory = cfg.dataDir; + ExecStart = "${cfg.package}/bin/${bin}"; + Restart = "always"; + RestartSec = "10"; + EnvironmentFile = lib.optional (cfg.environmentFile != null) cfg.environmentFile; + }; + }; +in +{ + options.services.minne = { + enable = lib.mkEnableOption "the Minne knowledge management server"; + + package = lib.mkOption { + type = lib.types.package; + default = + if self != null then + self.packages.${pkgs.stdenv.hostPlatform.system}.default + else + throw "services.minne.package must be set when the Minne flake is not available"; + defaultText = lib.literalExpression "minne.packages.\${system}.default"; + description = "Minne package providing the main/server/worker binaries."; + }; + + mode = lib.mkOption { + type = lib.types.enum [ + "combined" + "split" + ]; + default = "combined"; + description = '' + "combined" runs a single `main` process (API, web UI, and worker). + "split" runs the API-only `server` alongside a standalone `worker`. + ''; + }; + + user = lib.mkOption { + type = lib.types.str; + default = "minne"; + description = "User account under which Minne runs."; + }; + + group = lib.mkOption { + type = lib.types.str; + default = "minne"; + description = "Group under which Minne runs."; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 3000; + description = "TCP port the Minne HTTP server listens on (HTTP_PORT)."; + }; + + dataDir = lib.mkOption { + type = lib.types.path; + default = "/var/lib/minne"; + description = "State directory for Minne (DATA_DIR)."; + }; + + logLevel = lib.mkOption { + type = lib.types.str; + default = "info"; + description = "Value for RUST_LOG."; + }; + + openFirewall = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Open Minne `port` in the firewall."; + }; + + environment = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = { }; + example = { + EMBEDDING_BACKEND = "hashed"; + }; + description = "Extra environment variables passed to every Minne service."; + }; + + environmentFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + example = "/run/secrets/minne.env"; + description = '' + Path to a systemd EnvironmentFile holding secrets such as + `OPENAI_API_KEY`. SurrealDB credentials can be set via + `services.minne.surrealdb` or included here. + ''; + }; + + surrealdb = { + enable = lib.mkEnableOption "a bundled SurrealDB instance for Minne"; + + package = lib.mkOption { + type = lib.types.package; + default = import ../packages/surrealdb { inherit pkgs; }; + description = "SurrealDB package used when `services.minne.surrealdb.enable` is true."; + }; + + host = lib.mkOption { + type = lib.types.str; + default = "127.0.0.1"; + description = "Host SurrealDB binds to and Minne connects to."; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 8000; + description = "Port SurrealDB listens on and Minne connects to."; + }; + + dataDir = lib.mkOption { + type = lib.types.path; + default = "/var/lib/surrealdb"; + description = "RocksDB data directory for bundled SurrealDB."; + }; + + user = lib.mkOption { + type = lib.types.str; + default = "surrealdb"; + description = "User account under which bundled SurrealDB runs."; + }; + + group = lib.mkOption { + type = lib.types.str; + default = "surrealdb"; + description = "Group under which bundled SurrealDB runs."; + }; + + namespace = lib.mkOption { + type = lib.types.str; + default = "minne"; + description = "SurrealDB namespace passed to Minne (SURREALDB_NAMESPACE)."; + }; + + database = lib.mkOption { + type = lib.types.str; + default = "minne"; + description = "SurrealDB database passed to Minne (SURREALDB_DATABASE)."; + }; + + username = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + example = "root"; + description = '' + Root username for bundled SurrealDB (`--user`). When set, `password` + must also be set and Minne receives matching SURREALDB_USERNAME/PASSWORD. + ''; + }; + + password = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = "Root password for bundled SurrealDB (`--pass`)."; + }; + + credentialsFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + example = "/run/secrets/surrealdb-credentials"; + description = '' + Optional EnvironmentFile for bundled SurrealDB. Use when credentials + should not appear in the Nix store. Expects SURREALDB_USER and + SURREALDB_PASS (or SURREALDB_USERNAME / SURREALDB_PASSWORD). + ''; + }; + + openFirewall = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Open bundled SurrealDB `port` in the firewall."; + }; + + after = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ "network-online.target" ]; + description = "Extra units ordered before Minne when using an external SurrealDB."; + }; + + requires = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ "surrealdb.service" ]; + description = "Extra units Minne hard-depends on (external SurrealDB)."; + }; + }; + }; + + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = + !cfg.surrealdb.enable || cfg.surrealdb.username == null || cfg.surrealdb.password != null; + message = "services.minne.surrealdb.password must be set when username is set."; + } + ]; + + systemd.services = lib.mkMerge [ + (lib.mkIf cfg.surrealdb.enable { + surrealdb = { + description = "SurrealDB for Minne"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + serviceConfig = { + Type = "simple"; + User = cfg.surrealdb.user; + Group = cfg.surrealdb.group; + WorkingDirectory = cfg.surrealdb.dataDir; + ExecStart = "${cfg.surrealdb.package}/bin/surreal start --bind ${cfg.surrealdb.host}:${toString cfg.surrealdb.port} ${surrealAuthArgs} rocksdb:${cfg.surrealdb.dataDir}/data.db"; + Restart = "always"; + RestartSec = "10"; + EnvironmentFile = lib.optional ( + cfg.surrealdb.credentialsFile != null + ) cfg.surrealdb.credentialsFile; + }; + }; + }) + (lib.mkIf (cfg.mode == "combined") { + minne = mkService { + description = "Minne — API, web UI, and background worker"; + bin = "main"; + }; + }) + (lib.mkIf (cfg.mode == "split") { + minne-server = mkService { + description = "Minne — API-only server"; + bin = "server"; + }; + minne-worker = mkService { + description = "Minne — standalone background worker"; + bin = "worker"; + }; + }) + ]; + + users.users = lib.mkMerge [ + (lib.mkIf (cfg.user == "minne") { + minne = { + isSystemUser = true; + inherit (cfg) group; + home = cfg.dataDir; + createHome = true; + }; + }) + (lib.mkIf (cfg.surrealdb.enable && cfg.surrealdb.user == "surrealdb") { + surrealdb = { + isSystemUser = true; + inherit (cfg.surrealdb) group; + home = cfg.surrealdb.dataDir; + createHome = true; + }; + }) + ]; + + users.groups = lib.mkMerge [ + (lib.mkIf (cfg.group == "minne") { + minne = { }; + }) + (lib.mkIf (cfg.surrealdb.enable && cfg.surrealdb.group == "surrealdb") { + surrealdb = { }; + }) + ]; + + systemd.tmpfiles.rules = [ + "d ${cfg.dataDir} 0750 ${cfg.user} ${cfg.group} -" + ] + ++ lib.optional cfg.surrealdb.enable "d ${cfg.surrealdb.dataDir} 0750 ${cfg.surrealdb.user} ${cfg.surrealdb.group} -"; + + networking.firewall.allowedTCPPorts = + (lib.optional cfg.openFirewall cfg.port) + ++ lib.optional (cfg.surrealdb.enable && cfg.surrealdb.openFirewall) cfg.surrealdb.port; + }; +} diff --git a/nix/minne-release.nix b/nix/packages/release.nix similarity index 98% rename from nix/minne-release.nix rename to nix/packages/release.nix index 7c794c2..a5318c2 100644 --- a/nix/minne-release.nix +++ b/nix/packages/release.nix @@ -1,4 +1,3 @@ -# Portable release archives for cargo-dist-compatible GitHub Releases layout. { lib, stdenv, diff --git a/nix/packages/rust-toolchain.nix b/nix/packages/rust-toolchain.nix new file mode 100644 index 0000000..f99676e --- /dev/null +++ b/nix/packages/rust-toolchain.nix @@ -0,0 +1,38 @@ +{ + fenix, + system, + rustVersion, + manifestSha256, +}: +let + components = [ + "rustc" + "cargo" + "clippy" + "rustfmt" + "rust-analyzer" + ]; + targets = [ + "x86_64-unknown-linux-gnu" + "x86_64-pc-windows-msvc" + ]; + windowsTarget = "x86_64-pc-windows-msvc"; + + toolchainFile = builtins.toFile "rust-toolchain.toml" '' + [toolchain] + channel = "${rustVersion}" + components = ${builtins.toJSON components} + targets = ${builtins.toJSON targets} + ''; + + fenixPkgs = fenix.packages.${system}; + toolchain = fenixPkgs.fromToolchainFile { + file = toolchainFile; + sha256 = manifestSha256; + }; +in +{ + inherit rustVersion toolchain windowsTarget; + + mkCraneLib = _: craneLib: craneLib.overrideToolchain (_: toolchain); +} diff --git a/nix/surrealdb-binary.nix b/nix/packages/surrealdb/binary.nix similarity index 73% rename from nix/surrealdb-binary.nix rename to nix/packages/surrealdb/binary.nix index 6467348..8fb986f 100644 --- a/nix/surrealdb-binary.nix +++ b/nix/packages/surrealdb/binary.nix @@ -1,4 +1,3 @@ -# Pinned SurrealDB release binary (avoids building nixpkgs surrealdb from source). { lib, stdenv, @@ -6,14 +5,16 @@ autoPatchelfHook, openssl, rocksdb, + surrealdbVersion ? (import ../../versions.nix).surrealdbVersion, + surrealdbBinaryHash ? (import ../../versions.nix).surrealdbBinaryHash, }: stdenv.mkDerivation { pname = "surrealdb"; - version = "2.6.5"; + version = surrealdbVersion; src = fetchurl { - url = "https://github.com/surrealdb/surrealdb/releases/download/v2.6.5/surreal-v2.6.5.linux-amd64.tgz"; - hash = "sha256-kp1z9GxPtZ8jeBDm/m2lTBdWBk8+2NfR9qlw6P3zj7A="; + url = "https://github.com/surrealdb/surrealdb/releases/download/v${surrealdbVersion}/surreal-v${surrealdbVersion}.linux-amd64.tgz"; + hash = surrealdbBinaryHash; }; nativeBuildInputs = [ autoPatchelfHook ]; diff --git a/nix/packages/surrealdb/default.nix b/nix/packages/surrealdb/default.nix new file mode 100644 index 0000000..f2a6bde --- /dev/null +++ b/nix/packages/surrealdb/default.nix @@ -0,0 +1,5 @@ +{ + pkgs, + system ? pkgs.stdenv.hostPlatform.system, +}: +if system == "x86_64-linux" then pkgs.callPackage ./binary.nix { } else pkgs.surrealdb diff --git a/nix/windows-cross.nix b/nix/packages/windows-cross.nix similarity index 94% rename from nix/windows-cross.nix rename to nix/packages/windows-cross.nix index 51ff6ad..8bd5780 100644 --- a/nix/windows-cross.nix +++ b/nix/packages/windows-cross.nix @@ -1,6 +1,4 @@ -# Offline MSVC CRT + Windows SDK for cross-compiling to x86_64-pc-windows-msvc. { - lib, stdenv, xwin, cacert, @@ -9,9 +7,6 @@ }: let cmakeOverride = writeText "override.cmake" '' - # macOS paths usually start with /Users/*. Unfortunately, clang-cl interprets - # paths starting with /U as macro undefines, so we need to put a -- before the - # input file path to force it to be treated as a path. string(REPLACE "-c " "-c -- " CMAKE_C_COMPILE_OBJECT "''${CMAKE_C_COMPILE_OBJECT}") string(REPLACE "-c " "-c -- " CMAKE_CXX_COMPILE_OBJECT "''${CMAKE_CXX_COMPILE_OBJECT}") string(REPLACE "/D" "-D" CMAKE_RC_FLAGS "''${CMAKE_RC_FLAGS_INIT}") diff --git a/nix/rust-toolchain.nix b/nix/rust-toolchain.nix deleted file mode 100644 index 5831f57..0000000 --- a/nix/rust-toolchain.nix +++ /dev/null @@ -1,23 +0,0 @@ -# Shared Rust toolchain for dev shell, crane builds, and cross-compilation. -{ - fenix, - system, - toolchainFile, - # Manifest hash for https://static.rust-lang.org/dist/channel-rust-.toml - manifestSha256 ? "sha256-SDu4snEWjuZU475PERvu+iO50Mi39KVjqCeJeNvpguU=", -}: -let - fenixPkgs = fenix.packages.${system}; - toolchain = fenixPkgs.fromToolchainFile { - file = toolchainFile; - sha256 = manifestSha256; - }; - parsed = builtins.fromTOML (builtins.readFile toolchainFile); - rustVersion = parsed.toolchain.channel; - windowsTarget = "x86_64-pc-windows-msvc"; -in -{ - inherit rustVersion toolchain windowsTarget; - - mkCraneLib = pkgs: craneLib: craneLib.overrideToolchain (_: toolchain); -} diff --git a/nix/tests/module-eval.nix b/nix/tests/module-eval.nix new file mode 100644 index 0000000..78eb955 --- /dev/null +++ b/nix/tests/module-eval.nix @@ -0,0 +1,128 @@ +{ + lib, + pkgs, + minne-pkg, + surrealdbPkg, + minneNixosModule, + nixosSystem, +}: +let + evalModule = + module: + (nixosSystem { + inherit (pkgs.stdenv.hostPlatform) system; + modules = [ + minneNixosModule + { + nixpkgs.pkgs = pkgs; + boot.loader.grub.enable = false; + fileSystems."/".device = "/dev/sda1"; + system.stateVersion = "25.11"; + } + module + ]; + }).config; + + combined = evalModule { + services.minne = { + enable = true; + package = minne-pkg; + }; + }; + + split = evalModule { + services.minne = { + enable = true; + package = minne-pkg; + mode = "split"; + }; + }; + + external = evalModule { + services.minne = { + enable = true; + package = minne-pkg; + surrealdb = { + enable = false; + host = "db.internal"; + port = 9999; + after = [ "surrealdb.service" ]; + requires = [ "surrealdb.service" ]; + }; + }; + }; + + bundled = evalModule { + services.minne = { + enable = true; + package = minne-pkg; + surrealdb = { + enable = true; + package = surrealdbPkg; + }; + }; + }; + + hasUnit = cfg: name: cfg.systemd.services ? ${name}; + + assertions = [ + { + assertion = hasUnit combined "minne" && !(hasUnit combined "minne-server"); + message = "combined mode must define only the `minne` unit."; + } + { + assertion = combined.systemd.services.minne.serviceConfig.ExecStart == "${minne-pkg}/bin/main"; + message = "combined mode must run the `main` binary."; + } + { + assertion = + hasUnit split "minne-server" && hasUnit split "minne-worker" && !(hasUnit split "minne"); + message = "split mode must define `minne-server` and `minne-worker`, not `minne`."; + } + { + assertion = + split.systemd.services.minne-server.serviceConfig.ExecStart == "${minne-pkg}/bin/server"; + message = "split mode `minne-server` must run the `server` binary."; + } + { + assertion = + split.systemd.services.minne-worker.serviceConfig.ExecStart == "${minne-pkg}/bin/worker"; + message = "split mode `minne-worker` must run the `worker` binary."; + } + { + assertion = !(hasUnit external "surrealdb"); + message = "external SurrealDB must not define a bundled `surrealdb` unit."; + } + { + assertion = + external.systemd.services.minne.environment.SURREALDB_ADDRESS == "ws://db.internal:9999"; + message = "external SurrealDB host/port must be threaded into SURREALDB_ADDRESS."; + } + { + assertion = + lib.elem "surrealdb.service" external.systemd.services.minne.after + && lib.elem "surrealdb.service" external.systemd.services.minne.requires; + message = "external SurrealDB after/requires must order Minne after the DB unit."; + } + { + assertion = hasUnit bundled "surrealdb"; + message = "bundled SurrealDB must define the `surrealdb` unit."; + } + { + assertion = lib.hasPrefix "${surrealdbPkg}/bin/surreal" bundled.systemd.services.surrealdb.serviceConfig.ExecStart; + message = "bundled SurrealDB unit must use the overridden `surrealdb.package`."; + } + ]; + + failures = lib.filter (a: !a.assertion) assertions; +in +pkgs.runCommand "minne-module-eval-checks" { } ( + if failures == [ ] then + "touch $out" + else + '' + echo "services.minne module shape checks failed:" >&2 + ${lib.concatMapStringsSep "\n" (a: "echo ' - ${a.message}' >&2") failures} + exit 1 + '' +) diff --git a/nix/tests/vm-smoke.nix b/nix/tests/vm-smoke.nix new file mode 100644 index 0000000..e50440d --- /dev/null +++ b/nix/tests/vm-smoke.nix @@ -0,0 +1,44 @@ +{ + pkgs, + minne-pkg, + surrealdbPkg, + minneNixosModule, +}: +pkgs.testers.nixosTest { + name = "minne-smoke"; + + nodes.machine = + { ... }: + { + imports = [ minneNixosModule ]; + + virtualisation.memorySize = 4096; + + services.minne = { + enable = true; + package = minne-pkg; + surrealdb = { + enable = true; + package = surrealdbPkg; + username = "root_user"; + password = "root_password"; + namespace = "test"; + database = "test"; + }; + environment = { + OPENAI_API_KEY = "test-key"; + STORAGE = "local"; + EMBEDDING_BACKEND = "hashed"; + INDEX_REBUILD_INTERVAL_SECS = "0"; + }; + }; + }; + + testScript = '' + machine.wait_for_unit("surrealdb.service") + machine.wait_for_unit("minne.service") + machine.wait_for_open_port(3000) + machine.succeed("curl -sf http://127.0.0.1:3000/api/v1/live") + machine.succeed("curl -sf http://127.0.0.1:3000/api/v1/ready") + ''; +} diff --git a/nix/versions.nix b/nix/versions.nix new file mode 100644 index 0000000..0d4fdb2 --- /dev/null +++ b/nix/versions.nix @@ -0,0 +1,18 @@ +{ + minneVersion = "1.0.5"; + ortVersion = "1.23.2"; + rustVersion = "1.91.1"; + # Manifest hash for https://static.rust-lang.org/dist/channel-rust-.toml + rustManifestSha256 = "sha256-SDu4snEWjuZU475PERvu+iO50Mi39KVjqCeJeNvpguU="; + mozjsRelease = "mozjs-sys-v140.10.1-0"; + surrealdbVersion = "2.6.5"; + surrealdbBinaryHash = "sha256-kp1z9GxPtZ8jeBDm/m2lTBdWBk8+2NfR9qlw6P3zj7A="; + mozjsHashes = { + "x86_64-unknown-linux-gnu" = "sha256-e5kW8HTg6Hrd3sGgU9bqFNTTf7wJCChFOwKE3xyYT4Q="; + "aarch64-unknown-linux-gnu" = "sha256-VXrcktvjSH+14tO9Kzx+n9f/9ZQGAzfEsniiT+xKT6Q="; + "aarch64-apple-darwin" = "sha256-T3y73nVic6R60keUpmVRFe110Eh7AcE/VwZQWXRU9A0="; + "x86_64-apple-darwin" = "sha256-4v6f6c1OwYdg1FKnFfdLEsrRdyghcxup4gF7ioTZzm4="; + }; + mozjsArchiveWindowsHash = "sha256-nEX55a4vZJGxlDMCea9TEee60HiNe/yQzXtUqMlaM3c="; + ortArchiveWindowsHash = "sha256-CzjfmvIYNOQec9YC2Q21ywbb0cphiUi48dZtYHrJ880="; +} diff --git a/nix/vm-smoke-test.nix b/nix/vm-smoke-test.nix deleted file mode 100644 index 1f7e8bb..0000000 --- a/nix/vm-smoke-test.nix +++ /dev/null @@ -1,72 +0,0 @@ -# NixOS VM smoke test for the packaged Minne server. -{ - lib, - pkgs, - minne-pkg, - surrealdb, -}: -pkgs.testers.nixosTest { - name = "minne-smoke"; - - nodes.machine = { - virtualisation.memorySize = 4096; - - systemd.services.surrealdb = { - description = "SurrealDB for Minne smoke test"; - wantedBy = [ "multi-user.target" ]; - after = [ "network.target" ]; - preStart = "mkdir -p /var/lib/surrealdb"; - serviceConfig = { - Type = "simple"; - ExecStart = "${surrealdb}/bin/surreal start --bind 127.0.0.1:8000 --user root_user --pass root_password rocksdb:/var/lib/surrealdb/db"; - }; - }; - - systemd.services.minne = { - description = "Minne server smoke test"; - wantedBy = [ "multi-user.target" ]; - after = [ - "surrealdb.service" - "network.target" - ]; - requires = [ "surrealdb.service" ]; - preStart = '' - for i in $(seq 1 60); do - if ${pkgs.netcat}/bin/nc -z 127.0.0.1 8000; then - exit 0 - fi - sleep 1 - done - echo "surrealdb did not become ready on port 8000" >&2 - exit 1 - ''; - serviceConfig = { - Type = "simple"; - ExecStart = "${minne-pkg}/bin/main"; - Environment = [ - "SURREALDB_ADDRESS=ws://127.0.0.1:8000" - "SURREALDB_USERNAME=root_user" - "SURREALDB_PASSWORD=root_password" - "SURREALDB_NAMESPACE=test" - "SURREALDB_DATABASE=test" - "OPENAI_API_KEY=test-key" - "HTTP_PORT=3000" - "STORAGE=local" - "DATA_DIR=/var/lib/minne" - "EMBEDDING_BACKEND=hashed" - "RUST_LOG=info" - "INDEX_REBUILD_INTERVAL_SECS=0" - ]; - StateDirectory = "minne"; - }; - }; - }; - - testScript = '' - machine.wait_for_unit("surrealdb.service") - machine.wait_for_unit("minne.service") - machine.wait_for_open_port(3000) - machine.succeed("curl -sf http://127.0.0.1:3000/api/v1/live") - machine.succeed("curl -sf http://127.0.0.1:3000/api/v1/ready") - ''; -} diff --git a/rust-toolchain.toml b/rust-toolchain.toml deleted file mode 100644 index 366914a..0000000 --- a/rust-toolchain.toml +++ /dev/null @@ -1,4 +0,0 @@ -[toolchain] -channel = "1.91.1" -components = ["rustc", "cargo", "clippy", "rustfmt", "rust-analyzer"] -targets = ["x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"] diff --git a/rustfmt.toml b/rustfmt.toml deleted file mode 100644 index f216078..0000000 --- a/rustfmt.toml +++ /dev/null @@ -1 +0,0 @@ -edition = "2024"